You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2018/08/27 16:26:21 UTC
[01/51] [partial] incubator-airflow-site git commit: 1.10.0
Repository: incubator-airflow-site
Updated Branches:
refs/heads/asf-site 28a3eb600 -> 11437c14a
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/bigquery_hook.html
----------------------------------------------------------------------
diff --git a/_modules/bigquery_hook.html b/_modules/bigquery_hook.html
deleted file mode 100644
index e58a93b..0000000
--- a/_modules/bigquery_hook.html
+++ /dev/null
@@ -1,1279 +0,0 @@
-
-
-<!DOCTYPE html>
-<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
-<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
-<head>
- <meta charset="utf-8">
-
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
-
- <title>bigquery_hook — Airflow Documentation</title>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../genindex.html"/>
- <link rel="search" title="Search" href="../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../index.html"/>
- <link rel="up" title="Module code" href="index.html"/>
-
-
- <script src="../_static/js/modernizr.min.js"></script>
-
-</head>
-
-<body class="wy-body-for-nav" role="document">
-
-
- <div class="wy-grid-for-nav">
-
-
- <nav data-toggle="wy-nav-shift" class="wy-nav-side">
- <div class="wy-side-scroll">
- <div class="wy-side-nav-search">
-
-
-
- <a href="../index.html" class="icon icon-home"> Airflow
-
-
-
- </a>
-
-
-
-
-
-
-
-<div role="search">
- <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
- <input type="text" name="q" placeholder="Search docs" />
- <input type="hidden" name="check_keywords" value="yes" />
- <input type="hidden" name="area" value="default" />
- </form>
-</div>
-
-
- </div>
-
- <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
-
-
-
-
-
-
- <ul>
-<li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../configuration.html">Configuration</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../ui.html">UI / Screenshots</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../concepts.html">Concepts</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../profiling.html">Data Profiling</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../cli.html">Command Line Interface</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../scheduler.html">Scheduling & Triggers</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../plugins.html">Plugins</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../api.html">Experimental Rest API</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../integration.html">Integration</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../code.html">API Reference</a></li>
-</ul>
-
-
-
- </div>
- </div>
- </nav>
-
- <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
-
-
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
-
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="../index.html">Airflow</a>
-
- </nav>
-
-
-
- <div class="wy-nav-content">
- <div class="rst-content">
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-<div role="navigation" aria-label="breadcrumbs navigation">
-
- <ul class="wy-breadcrumbs">
-
- <li><a href="../index.html">Docs</a> »</li>
-
- <li><a href="index.html">Module code</a> »</li>
-
- <li>bigquery_hook</li>
-
-
- <li class="wy-breadcrumbs-aside">
-
-
-
- </li>
-
- </ul>
-
-
- <hr/>
-</div>
- <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
- <div itemprop="articleBody">
-
- <h1>Source code for bigquery_hook</h1><div class="highlight"><pre>
-<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
-<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
-<span class="c1">#</span>
-
-<span class="sd">"""</span>
-<span class="sd">This module contains a BigQuery Hook, as well as a very basic PEP 249</span>
-<span class="sd">implementation for BigQuery.</span>
-<span class="sd">"""</span>
-
-<span class="kn">import</span> <span class="nn">time</span>
-
-<span class="kn">from</span> <span class="nn">apiclient.discovery</span> <span class="k">import</span> <span class="n">build</span><span class="p">,</span> <span class="n">HttpError</span>
-<span class="kn">from</span> <span class="nn">googleapiclient</span> <span class="k">import</span> <span class="n">errors</span>
-<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">range</span>
-<span class="kn">from</span> <span class="nn">pandas_gbq.gbq</span> <span class="k">import</span> <span class="n">GbqConnector</span><span class="p">,</span> \
- <span class="n">_parse_data</span> <span class="k">as</span> <span class="n">gbq_parse_data</span><span class="p">,</span> \
- <span class="n">_check_google_client_version</span> <span class="k">as</span> <span class="n">gbq_check_google_client_version</span><span class="p">,</span> \
- <span class="n">_test_google_api_imports</span> <span class="k">as</span> <span class="n">gbq_test_google_api_imports</span>
-<span class="kn">from</span> <span class="nn">pandas.tools.merge</span> <span class="k">import</span> <span class="n">concat</span>
-<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="k">import</span> <span class="n">basestring</span>
-
-<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="k">import</span> <span class="n">GoogleCloudBaseHook</span>
-<span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="k">import</span> <span class="n">DbApiHook</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
-
-
-<div class="viewcode-block" id="BigQueryHook"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.BigQueryHook">[docs]</a><span class="k">class</span> <span class="nc">BigQueryHook</span><span class="p">(</span><span class="n">GoogleCloudBaseHook</span><span class="p">,</span> <span class="n">DbApiHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Interact with BigQuery. This hook uses the Google Cloud Platform</span>
-<span class="sd"> connection.</span>
-<span class="sd"> """</span>
- <span class="n">conn_name_attr</span> <span class="o">=</span> <span class="s1">'bigquery_conn_id'</span>
-
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
- <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">'bigquery_default'</span><span class="p">,</span>
- <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
- <span class="n">conn_id</span><span class="o">=</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
- <span class="n">delegate_to</span><span class="o">=</span><span class="n">delegate_to</span><span class="p">)</span>
-
-<div class="viewcode-block" id="BigQueryHook.get_conn"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.BigQueryHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Returns a BigQuery PEP 249 connection object.</span>
-<span class="sd"> """</span>
- <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_service</span><span class="p">()</span>
- <span class="n">project</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_field</span><span class="p">(</span><span class="s1">'project'</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">BigQueryConnection</span><span class="p">(</span><span class="n">service</span><span class="o">=</span><span class="n">service</span><span class="p">,</span> <span class="n">project_id</span><span class="o">=</span><span class="n">project</span><span class="p">)</span></div>
-
-<div class="viewcode-block" id="BigQueryHook.get_service"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.BigQueryHook.get_service">[docs]</a> <span class="k">def</span> <span class="nf">get_service</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Returns a BigQuery service object.</span>
-<span class="sd"> """</span>
- <span class="n">http_authorized</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
- <span class="k">return</span> <span class="n">build</span><span class="p">(</span><span class="s1">'bigquery'</span><span class="p">,</span> <span class="s1">'v2'</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">)</span></div>
-
-<div class="viewcode-block" id="BigQueryHook.insert_rows"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.BigQueryHook.insert_rows">[docs]</a> <span class="k">def</span> <span class="nf">insert_rows</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">rows</span><span class="p">,</span> <span class="n">target_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">commit_every</span><span class="o">=</span><span class="mi">1000</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Insertion is currently unsupported. Theoretically, you could use</span>
-<span class="sd"> BigQuery's streaming API to insert rows into a table, but this hasn't</span>
-<span class="sd"> been implemented.</span>
-<span class="sd"> """</span>
- <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">()</span></div>
-
-<div class="viewcode-block" id="BigQueryHook.get_pandas_df"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.BigQueryHook.get_pandas_df">[docs]</a> <span class="k">def</span> <span class="nf">get_pandas_df</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">dialect</span><span class="o">=</span><span class="s1">'legacy'</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Returns a Pandas DataFrame for the results produced by a BigQuery</span>
-<span class="sd"> query. The DbApiHook method must be overridden because Pandas</span>
-<span class="sd"> doesn't support PEP 249 connections, except for SQLite. See:</span>
-
-<span class="sd"> https://github.com/pydata/pandas/blob/master/pandas/io/sql.py#L447</span>
-<span class="sd"> https://github.com/pydata/pandas/issues/6900</span>
-
-<span class="sd"> :param bql: The BigQuery SQL to execute.</span>
-<span class="sd"> :type bql: string</span>
-<span class="sd"> :param parameters: The parameters to render the SQL query with (not used, leave to override superclass method)</span>
-<span class="sd"> :type parameters: mapping or iterable</span>
-<span class="sd"> :param dialect: Dialect of BigQuery SQL – legacy SQL or standard SQL</span>
-<span class="sd"> :type dialect: string in {'legacy', 'standard'}, default 'legacy'</span>
-<span class="sd"> """</span>
- <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_service</span><span class="p">()</span>
- <span class="n">project</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_field</span><span class="p">(</span><span class="s1">'project'</span><span class="p">)</span>
- <span class="n">connector</span> <span class="o">=</span> <span class="n">BigQueryPandasConnector</span><span class="p">(</span><span class="n">project</span><span class="p">,</span> <span class="n">service</span><span class="p">,</span> <span class="n">dialect</span><span class="o">=</span><span class="n">dialect</span><span class="p">)</span>
- <span class="n">schema</span><span class="p">,</span> <span class="n">pages</span> <span class="o">=</span> <span class="n">connector</span><span class="o">.</span><span class="n">run_query</span><span class="p">(</span><span class="n">bql</span><span class="p">)</span>
- <span class="n">dataframe_list</span> <span class="o">=</span> <span class="p">[]</span>
-
- <span class="k">while</span> <span class="nb">len</span><span class="p">(</span><span class="n">pages</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
- <span class="n">page</span> <span class="o">=</span> <span class="n">pages</span><span class="o">.</span><span class="n">pop</span><span class="p">()</span>
- <span class="n">dataframe_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">gbq_parse_data</span><span class="p">(</span><span class="n">schema</span><span class="p">,</span> <span class="n">page</span><span class="p">))</span>
-
- <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">dataframe_list</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">concat</span><span class="p">(</span><span class="n">dataframe_list</span><span class="p">,</span> <span class="n">ignore_index</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">gbq_parse_data</span><span class="p">(</span><span class="n">schema</span><span class="p">,</span> <span class="p">[])</span></div>
-
-<div class="viewcode-block" id="BigQueryHook.table_exists"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.BigQueryHook.table_exists">[docs]</a> <span class="k">def</span> <span class="nf">table_exists</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Checks for the existence of a table in Google BigQuery.</span>
-
-<span class="sd"> :param project_id: The Google cloud project in which to look for the table. The connection supplied to the hook</span>
-<span class="sd"> must provide access to the specified project.</span>
-<span class="sd"> :type project_id: string</span>
-<span class="sd"> :param dataset_id: The name of the dataset in which to look for the table.</span>
-<span class="sd"> storage bucket.</span>
-<span class="sd"> :type dataset_id: string</span>
-<span class="sd"> :param table_id: The name of the table to check the existence of.</span>
-<span class="sd"> :type table_id: string</span>
-<span class="sd"> """</span>
- <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_service</span><span class="p">()</span>
- <span class="k">try</span><span class="p">:</span>
- <span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
- <span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
- <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
- <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span>
- <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
- <span class="k">return</span> <span class="kc">True</span>
- <span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
- <span class="k">if</span> <span class="n">e</span><span class="o">.</span><span class="n">resp</span><span class="p">[</span><span class="s1">'status'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'404'</span><span class="p">:</span>
- <span class="k">return</span> <span class="kc">False</span>
- <span class="k">raise</span></div></div>
-
-
-<span class="k">class</span> <span class="nc">BigQueryPandasConnector</span><span class="p">(</span><span class="n">GbqConnector</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> This connector behaves identically to GbqConnector (from Pandas), except</span>
-<span class="sd"> that it allows the service to be injected, and disables a call to</span>
-<span class="sd"> self.get_credentials(). This allows Airflow to use BigQuery with Pandas</span>
-<span class="sd"> without forcing a three legged OAuth connection. Instead, we can inject</span>
-<span class="sd"> service account credentials into the binding.</span>
-<span class="sd"> """</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">service</span><span class="p">,</span> <span class="n">reauth</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">dialect</span><span class="o">=</span><span class="s1">'legacy'</span><span class="p">):</span>
- <span class="n">gbq_check_google_client_version</span><span class="p">()</span>
- <span class="n">gbq_test_google_api_imports</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">reauth</span> <span class="o">=</span> <span class="n">reauth</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">service</span> <span class="o">=</span> <span class="n">service</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">dialect</span> <span class="o">=</span> <span class="n">dialect</span>
-
-
-<span class="k">class</span> <span class="nc">BigQueryConnection</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> BigQuery does not have a notion of a persistent connection. Thus, these</span>
-<span class="sd"> objects are small stateless factories for cursors, which do all the real</span>
-<span class="sd"> work.</span>
-<span class="sd"> """</span>
-
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">_args</span> <span class="o">=</span> <span class="n">args</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">_kwargs</span> <span class="o">=</span> <span class="n">kwargs</span>
-
- <span class="k">def</span> <span class="nf">close</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="sd">""" BigQueryConnection does not have anything to close. """</span>
- <span class="k">pass</span>
-
- <span class="k">def</span> <span class="nf">commit</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="sd">""" BigQueryConnection does not support transactions. """</span>
- <span class="k">pass</span>
-
- <span class="k">def</span> <span class="nf">cursor</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="sd">""" Return a new :py:class:`Cursor` object using the connection. """</span>
- <span class="k">return</span> <span class="n">BigQueryCursor</span><span class="p">(</span><span class="o">*</span><span class="bp">self</span><span class="o">.</span><span class="n">_args</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">_kwargs</span><span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">rollback</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span>
- <span class="s2">"BigQueryConnection does not have transactions"</span><span class="p">)</span>
-
-
-<span class="k">class</span> <span class="nc">BigQueryBaseCursor</span><span class="p">(</span><span class="n">LoggingMixin</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> The BigQuery base cursor contains helper methods to execute queries against</span>
-<span class="sd"> BigQuery. The methods can be used directly by operators, in cases where a</span>
-<span class="sd"> PEP 249 cursor isn't needed.</span>
-<span class="sd"> """</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">service</span><span class="p">,</span> <span class="n">project_id</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">service</span> <span class="o">=</span> <span class="n">service</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
-
- <span class="k">def</span> <span class="nf">run_query</span><span class="p">(</span>
- <span class="bp">self</span><span class="p">,</span> <span class="n">bql</span><span class="p">,</span> <span class="n">destination_dataset_table</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
- <span class="n">write_disposition</span> <span class="o">=</span> <span class="s1">'WRITE_EMPTY'</span><span class="p">,</span>
- <span class="n">allow_large_results</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">udf_config</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
- <span class="n">use_legacy_sql</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
- <span class="n">maximum_billing_tier</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">create_disposition</span><span class="o">=</span><span class="s1">'CREATE_IF_NEEDED'</span><span class="p">,</span>
- <span class="n">query_params</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Executes a BigQuery SQL query. Optionally persists results in a BigQuery</span>
-<span class="sd"> table. See here:</span>
-
-<span class="sd"> https://cloud.google.com/bigquery/docs/reference/v2/jobs</span>
-
-<span class="sd"> For more details about these parameters.</span>
-
-<span class="sd"> :param bql: The BigQuery SQL to execute.</span>
-<span class="sd"> :type bql: string</span>
-<span class="sd"> :param destination_dataset_table: The dotted <dataset>.<table></span>
-<span class="sd"> BigQuery table to save the query results.</span>
-<span class="sd"> :param write_disposition: What to do if the table already exists in</span>
-<span class="sd"> BigQuery.</span>
-<span class="sd"> :type write_disposition: string</span>
-<span class="sd"> :param create_disposition: Specifies whether the job is allowed to create new tables.</span>
-<span class="sd"> :type create_disposition: string</span>
-<span class="sd"> :param allow_large_results: Whether to allow large results.</span>
-<span class="sd"> :type allow_large_results: boolean</span>
-<span class="sd"> :param udf_config: The User Defined Function configuration for the query.</span>
-<span class="sd"> See https://cloud.google.com/bigquery/user-defined-functions for details.</span>
-<span class="sd"> :type udf_config: list</span>
-<span class="sd"> :param use_legacy_sql: Whether to use legacy SQL (true) or standard SQL (false).</span>
-<span class="sd"> :type use_legacy_sql: boolean</span>
-<span class="sd"> :param maximum_billing_tier: Positive integer that serves as a multiplier of the basic price.</span>
-<span class="sd"> :type maximum_billing_tier: integer</span>
-<span class="sd"> """</span>
- <span class="n">configuration</span> <span class="o">=</span> <span class="p">{</span>
- <span class="s1">'query'</span><span class="p">:</span> <span class="p">{</span>
- <span class="s1">'query'</span><span class="p">:</span> <span class="n">bql</span><span class="p">,</span>
- <span class="s1">'useLegacySql'</span><span class="p">:</span> <span class="n">use_legacy_sql</span><span class="p">,</span>
- <span class="s1">'maximumBillingTier'</span><span class="p">:</span> <span class="n">maximum_billing_tier</span>
- <span class="p">}</span>
- <span class="p">}</span>
-
- <span class="k">if</span> <span class="n">destination_dataset_table</span><span class="p">:</span>
- <span class="k">assert</span> <span class="s1">'.'</span> <span class="ow">in</span> <span class="n">destination_dataset_table</span><span class="p">,</span> <span class="p">(</span>
- <span class="s1">'Expected destination_dataset_table in the format of '</span>
- <span class="s1">'<dataset>.<table>. Got: </span><span class="si">{}</span><span class="s1">'</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">destination_dataset_table</span><span class="p">)</span>
- <span class="n">destination_project</span><span class="p">,</span> <span class="n">destination_dataset</span><span class="p">,</span> <span class="n">destination_table</span> <span class="o">=</span> \
- <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">destination_dataset_table</span><span class="p">,</span>
- <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">)</span>
- <span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">({</span>
- <span class="s1">'allowLargeResults'</span><span class="p">:</span> <span class="n">allow_large_results</span><span class="p">,</span>
- <span class="s1">'writeDisposition'</span><span class="p">:</span> <span class="n">write_disposition</span><span class="p">,</span>
- <span class="s1">'createDisposition'</span><span class="p">:</span> <span class="n">create_disposition</span><span class="p">,</span>
- <span class="s1">'destinationTable'</span><span class="p">:</span> <span class="p">{</span>
- <span class="s1">'projectId'</span><span class="p">:</span> <span class="n">destination_project</span><span class="p">,</span>
- <span class="s1">'datasetId'</span><span class="p">:</span> <span class="n">destination_dataset</span><span class="p">,</span>
- <span class="s1">'tableId'</span><span class="p">:</span> <span class="n">destination_table</span><span class="p">,</span>
- <span class="p">}</span>
- <span class="p">})</span>
- <span class="k">if</span> <span class="n">udf_config</span><span class="p">:</span>
- <span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">udf_config</span><span class="p">,</span> <span class="nb">list</span><span class="p">)</span>
- <span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">({</span>
- <span class="s1">'userDefinedFunctionResources'</span><span class="p">:</span> <span class="n">udf_config</span>
- <span class="p">})</span>
-
- <span class="k">if</span> <span class="n">query_params</span><span class="p">:</span>
- <span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">][</span><span class="s1">'queryParameters'</span><span class="p">]</span> <span class="o">=</span> <span class="n">query_params</span>
-
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_with_configuration</span><span class="p">(</span><span class="n">configuration</span><span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">run_extract</span><span class="p">(</span> <span class="c1"># noqa</span>
- <span class="bp">self</span><span class="p">,</span> <span class="n">source_project_dataset_table</span><span class="p">,</span> <span class="n">destination_cloud_storage_uris</span><span class="p">,</span>
- <span class="n">compression</span><span class="o">=</span><span class="s1">'NONE'</span><span class="p">,</span> <span class="n">export_format</span><span class="o">=</span><span class="s1">'CSV'</span><span class="p">,</span> <span class="n">field_delimiter</span><span class="o">=</span><span class="s1">','</span><span class="p">,</span>
- <span class="n">print_header</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Executes a BigQuery extract command to copy data from BigQuery to</span>
-<span class="sd"> Google Cloud Storage. See here:</span>
-
-<span class="sd"> https://cloud.google.com/bigquery/docs/reference/v2/jobs</span>
-
-<span class="sd"> For more details about these parameters.</span>
-
-<span class="sd"> :param source_project_dataset_table: The dotted <dataset>.<table></span>
-<span class="sd"> BigQuery table to use as the source data.</span>
-<span class="sd"> :type source_project_dataset_table: string</span>
-<span class="sd"> :param destination_cloud_storage_uris: The destination Google Cloud</span>
-<span class="sd"> Storage URI (e.g. gs://some-bucket/some-file.txt). Follows</span>
-<span class="sd"> convention defined here:</span>
-<span class="sd"> https://cloud.google.com/bigquery/exporting-data-from-bigquery#exportingmultiple</span>
-<span class="sd"> :type destination_cloud_storage_uris: list</span>
-<span class="sd"> :param compression: Type of compression to use.</span>
-<span class="sd"> :type compression: string</span>
-<span class="sd"> :param export_format: File format to export.</span>
-<span class="sd"> :type export_format: string</span>
-<span class="sd"> :param field_delimiter: The delimiter to use when extracting to a CSV.</span>
-<span class="sd"> :type field_delimiter: string</span>
-<span class="sd"> :param print_header: Whether to print a header for a CSV file extract.</span>
-<span class="sd"> :type print_header: boolean</span>
-<span class="sd"> """</span>
-
- <span class="n">source_project</span><span class="p">,</span> <span class="n">source_dataset</span><span class="p">,</span> <span class="n">source_table</span> <span class="o">=</span> \
- <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">source_project_dataset_table</span><span class="p">,</span>
- <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
- <span class="n">var_name</span><span class="o">=</span><span class="s1">'source_project_dataset_table'</span><span class="p">)</span>
-
- <span class="n">configuration</span> <span class="o">=</span> <span class="p">{</span>
- <span class="s1">'extract'</span><span class="p">:</span> <span class="p">{</span>
- <span class="s1">'sourceTable'</span><span class="p">:</span> <span class="p">{</span>
- <span class="s1">'projectId'</span><span class="p">:</span> <span class="n">source_project</span><span class="p">,</span>
- <span class="s1">'datasetId'</span><span class="p">:</span> <span class="n">source_dataset</span><span class="p">,</span>
- <span class="s1">'tableId'</span><span class="p">:</span> <span class="n">source_table</span><span class="p">,</span>
- <span class="p">},</span>
- <span class="s1">'compression'</span><span class="p">:</span> <span class="n">compression</span><span class="p">,</span>
- <span class="s1">'destinationUris'</span><span class="p">:</span> <span class="n">destination_cloud_storage_uris</span><span class="p">,</span>
- <span class="s1">'destinationFormat'</span><span class="p">:</span> <span class="n">export_format</span><span class="p">,</span>
- <span class="p">}</span>
- <span class="p">}</span>
-
- <span class="k">if</span> <span class="n">export_format</span> <span class="o">==</span> <span class="s1">'CSV'</span><span class="p">:</span>
- <span class="c1"># Only set fieldDelimiter and printHeader fields if using CSV.</span>
- <span class="c1"># Google does not like it if you set these fields for other export</span>
- <span class="c1"># formats.</span>
- <span class="n">configuration</span><span class="p">[</span><span class="s1">'extract'</span><span class="p">][</span><span class="s1">'fieldDelimiter'</span><span class="p">]</span> <span class="o">=</span> <span class="n">field_delimiter</span>
- <span class="n">configuration</span><span class="p">[</span><span class="s1">'extract'</span><span class="p">][</span><span class="s1">'printHeader'</span><span class="p">]</span> <span class="o">=</span> <span class="n">print_header</span>
-
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_with_configuration</span><span class="p">(</span><span class="n">configuration</span><span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">run_copy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
- <span class="n">source_project_dataset_tables</span><span class="p">,</span>
- <span class="n">destination_project_dataset_table</span><span class="p">,</span>
- <span class="n">write_disposition</span><span class="o">=</span><span class="s1">'WRITE_EMPTY'</span><span class="p">,</span>
- <span class="n">create_disposition</span><span class="o">=</span><span class="s1">'CREATE_IF_NEEDED'</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Executes a BigQuery copy command to copy data from one BigQuery table</span>
-<span class="sd"> to another. See here:</span>
-
-<span class="sd"> https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.copy</span>
-
-<span class="sd"> For more details about these parameters.</span>
-
-<span class="sd"> :param source_project_dataset_tables: One or more dotted</span>
-<span class="sd"> (project:|project.)<dataset>.<table></span>
-<span class="sd"> BigQuery tables to use as the source data. Use a list if there are</span>
-<span class="sd"> multiple source tables.</span>
-<span class="sd"> If <project> is not included, project will be the project defined</span>
-<span class="sd"> in the connection json.</span>
-<span class="sd"> :type source_project_dataset_tables: list|string</span>
-<span class="sd"> :param destination_project_dataset_table: The destination BigQuery</span>
-<span class="sd"> table. Format is: (project:|project.)<dataset>.<table></span>
-<span class="sd"> :type destination_project_dataset_table: string</span>
-<span class="sd"> :param write_disposition: The write disposition if the table already exists.</span>
-<span class="sd"> :type write_disposition: string</span>
-<span class="sd"> :param create_disposition: The create disposition if the table doesn't exist.</span>
-<span class="sd"> :type create_disposition: string</span>
-<span class="sd"> """</span>
- <span class="n">source_project_dataset_tables</span> <span class="o">=</span> <span class="p">(</span>
- <span class="p">[</span><span class="n">source_project_dataset_tables</span><span class="p">]</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">source_project_dataset_tables</span><span class="p">,</span> <span class="nb">list</span><span class="p">)</span>
- <span class="k">else</span> <span class="n">source_project_dataset_tables</span><span class="p">)</span>
-
- <span class="n">source_project_dataset_tables_fixup</span> <span class="o">=</span> <span class="p">[]</span>
- <span class="k">for</span> <span class="n">source_project_dataset_table</span> <span class="ow">in</span> <span class="n">source_project_dataset_tables</span><span class="p">:</span>
- <span class="n">source_project</span><span class="p">,</span> <span class="n">source_dataset</span><span class="p">,</span> <span class="n">source_table</span> <span class="o">=</span> \
- <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">source_project_dataset_table</span><span class="p">,</span>
- <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
- <span class="n">var_name</span><span class="o">=</span><span class="s1">'source_project_dataset_table'</span><span class="p">)</span>
- <span class="n">source_project_dataset_tables_fixup</span><span class="o">.</span><span class="n">append</span><span class="p">({</span>
- <span class="s1">'projectId'</span><span class="p">:</span> <span class="n">source_project</span><span class="p">,</span>
- <span class="s1">'datasetId'</span><span class="p">:</span> <span class="n">source_dataset</span><span class="p">,</span>
- <span class="s1">'tableId'</span><span class="p">:</span> <span class="n">source_table</span>
- <span class="p">})</span>
-
- <span class="n">destination_project</span><span class="p">,</span> <span class="n">destination_dataset</span><span class="p">,</span> <span class="n">destination_table</span> <span class="o">=</span> \
- <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">destination_project_dataset_table</span><span class="p">,</span>
- <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">)</span>
- <span class="n">configuration</span> <span class="o">=</span> <span class="p">{</span>
- <span class="s1">'copy'</span><span class="p">:</span> <span class="p">{</span>
- <span class="s1">'createDisposition'</span><span class="p">:</span> <span class="n">create_disposition</span><span class="p">,</span>
- <span class="s1">'writeDisposition'</span><span class="p">:</span> <span class="n">write_disposition</span><span class="p">,</span>
- <span class="s1">'sourceTables'</span><span class="p">:</span> <span class="n">source_project_dataset_tables_fixup</span><span class="p">,</span>
- <span class="s1">'destinationTable'</span><span class="p">:</span> <span class="p">{</span>
- <span class="s1">'projectId'</span><span class="p">:</span> <span class="n">destination_project</span><span class="p">,</span>
- <span class="s1">'datasetId'</span><span class="p">:</span> <span class="n">destination_dataset</span><span class="p">,</span>
- <span class="s1">'tableId'</span><span class="p">:</span> <span class="n">destination_table</span>
- <span class="p">}</span>
- <span class="p">}</span>
- <span class="p">}</span>
-
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_with_configuration</span><span class="p">(</span><span class="n">configuration</span><span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">run_load</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
- <span class="n">destination_project_dataset_table</span><span class="p">,</span>
- <span class="n">schema_fields</span><span class="p">,</span> <span class="n">source_uris</span><span class="p">,</span>
- <span class="n">source_format</span><span class="o">=</span><span class="s1">'CSV'</span><span class="p">,</span>
- <span class="n">create_disposition</span><span class="o">=</span><span class="s1">'CREATE_IF_NEEDED'</span><span class="p">,</span>
- <span class="n">skip_leading_rows</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
- <span class="n">write_disposition</span><span class="o">=</span><span class="s1">'WRITE_EMPTY'</span><span class="p">,</span>
- <span class="n">field_delimiter</span><span class="o">=</span><span class="s1">','</span><span class="p">,</span>
- <span class="n">max_bad_records</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
- <span class="n">quote_character</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">allow_quoted_newlines</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">allow_jagged_rows</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">schema_update_options</span><span class="o">=</span><span class="p">(),</span>
- <span class="n">src_fmt_configs</span><span class="o">=</span><span class="p">{}):</span>
- <span class="sd">"""</span>
-<span class="sd"> Executes a BigQuery load command to load data from Google Cloud Storage</span>
-<span class="sd"> to BigQuery. See here:</span>
-
-<span class="sd"> https://cloud.google.com/bigquery/docs/reference/v2/jobs</span>
-
-<span class="sd"> For more details about these parameters.</span>
-
-<span class="sd"> :param destination_project_dataset_table:</span>
-<span class="sd"> The dotted (<project>.|<project>:)<dataset>.<table> BigQuery table to load</span>
-<span class="sd"> data into. If <project> is not included, project will be the project defined</span>
-<span class="sd"> in the connection json.</span>
-<span class="sd"> :type destination_project_dataset_table: string</span>
-<span class="sd"> :param schema_fields: The schema field list as defined here:</span>
-<span class="sd"> https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load</span>
-<span class="sd"> :type schema_fields: list</span>
-<span class="sd"> :param source_uris: The source Google Cloud</span>
-<span class="sd"> Storage URI (e.g. gs://some-bucket/some-file.txt). A single wild</span>
-<span class="sd"> per-object name can be used.</span>
-<span class="sd"> :type source_uris: list</span>
-<span class="sd"> :param source_format: File format to export.</span>
-<span class="sd"> :type source_format: string</span>
-<span class="sd"> :param create_disposition: The create disposition if the table doesn't exist.</span>
-<span class="sd"> :type create_disposition: string</span>
-<span class="sd"> :param skip_leading_rows: Number of rows to skip when loading from a CSV.</span>
-<span class="sd"> :type skip_leading_rows: int</span>
-<span class="sd"> :param write_disposition: The write disposition if the table already exists.</span>
-<span class="sd"> :type write_disposition: string</span>
-<span class="sd"> :param field_delimiter: The delimiter to use when loading from a CSV.</span>
-<span class="sd"> :type field_delimiter: string</span>
-<span class="sd"> :param max_bad_records: The maximum number of bad records that BigQuery can</span>
-<span class="sd"> ignore when running the job.</span>
-<span class="sd"> :type max_bad_records: int</span>
-<span class="sd"> :param quote_character: The value that is used to quote data sections in a CSV file.</span>
-<span class="sd"> :type quote_character: string</span>
-<span class="sd"> :param allow_quoted_newlines: Whether to allow quoted newlines (true) or not (false).</span>
-<span class="sd"> :type allow_quoted_newlines: boolean</span>
-<span class="sd"> :param allow_jagged_rows: Accept rows that are missing trailing optional columns.</span>
-<span class="sd"> The missing values are treated as nulls. If false, records with missing trailing columns</span>
-<span class="sd"> are treated as bad records, and if there are too many bad records, an invalid error is</span>
-<span class="sd"> returned in the job result. Only applicable when soure_format is CSV.</span>
-<span class="sd"> :type allow_jagged_rows: bool</span>
-<span class="sd"> :param schema_update_options: Allows the schema of the desitination</span>
-<span class="sd"> table to be updated as a side effect of the load job.</span>
-<span class="sd"> :type schema_update_options: list</span>
-<span class="sd"> :param src_fmt_configs: configure optional fields specific to the source format</span>
-<span class="sd"> :type src_fmt_configs: dict</span>
-<span class="sd"> """</span>
-
- <span class="c1"># bigquery only allows certain source formats</span>
- <span class="c1"># we check to make sure the passed source format is valid</span>
- <span class="c1"># if it's not, we raise a ValueError</span>
- <span class="c1"># Refer to this link for more details:</span>
- <span class="c1"># https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.query.tableDefinitions.(key).sourceFormat</span>
- <span class="n">source_format</span> <span class="o">=</span> <span class="n">source_format</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
- <span class="n">allowed_formats</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"CSV"</span><span class="p">,</span> <span class="s2">"NEWLINE_DELIMITED_JSON"</span><span class="p">,</span> <span class="s2">"AVRO"</span><span class="p">,</span> <span class="s2">"GOOGLE_SHEETS"</span><span class="p">,</span> <span class="s2">"DATASTORE_BACKUP"</span><span class="p">]</span>
- <span class="k">if</span> <span class="n">source_format</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">allowed_formats</span><span class="p">:</span>
- <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"</span><span class="si">{0}</span><span class="s2"> is not a valid source format. "</span>
- <span class="s2">"Please use one of the following types: </span><span class="si">{1}</span><span class="s2">"</span>
- <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">source_format</span><span class="p">,</span> <span class="n">allowed_formats</span><span class="p">))</span>
-
- <span class="c1"># bigquery also allows you to define how you want a table's schema to change</span>
- <span class="c1"># as a side effect of a load</span>
- <span class="c1"># for more details:</span>
- <span class="c1"># https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load.schemaUpdateOptions</span>
- <span class="n">allowed_schema_update_options</span> <span class="o">=</span> <span class="p">[</span>
- <span class="s1">'ALLOW_FIELD_ADDITION'</span><span class="p">,</span>
- <span class="s2">"ALLOW_FIELD_RELAXATION"</span>
- <span class="p">]</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="nb">set</span><span class="p">(</span><span class="n">allowed_schema_update_options</span><span class="p">)</span><span class="o">.</span><span class="n">issuperset</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">)):</span>
- <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
- <span class="s2">"</span><span class="si">{0}</span><span class="s2"> contains invalid schema update options. "</span>
- <span class="s2">"Please only use one or more of the following options: </span><span class="si">{1}</span><span class="s2">"</span>
- <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">,</span> <span class="n">allowed_schema_update_options</span><span class="p">)</span>
- <span class="p">)</span>
-
- <span class="n">destination_project</span><span class="p">,</span> <span class="n">destination_dataset</span><span class="p">,</span> <span class="n">destination_table</span> <span class="o">=</span> \
- <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">destination_project_dataset_table</span><span class="p">,</span>
- <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
- <span class="n">var_name</span><span class="o">=</span><span class="s1">'destination_project_dataset_table'</span><span class="p">)</span>
-
- <span class="n">configuration</span> <span class="o">=</span> <span class="p">{</span>
- <span class="s1">'load'</span><span class="p">:</span> <span class="p">{</span>
- <span class="s1">'createDisposition'</span><span class="p">:</span> <span class="n">create_disposition</span><span class="p">,</span>
- <span class="s1">'destinationTable'</span><span class="p">:</span> <span class="p">{</span>
- <span class="s1">'projectId'</span><span class="p">:</span> <span class="n">destination_project</span><span class="p">,</span>
- <span class="s1">'datasetId'</span><span class="p">:</span> <span class="n">destination_dataset</span><span class="p">,</span>
- <span class="s1">'tableId'</span><span class="p">:</span> <span class="n">destination_table</span><span class="p">,</span>
- <span class="p">},</span>
- <span class="s1">'sourceFormat'</span><span class="p">:</span> <span class="n">source_format</span><span class="p">,</span>
- <span class="s1">'sourceUris'</span><span class="p">:</span> <span class="n">source_uris</span><span class="p">,</span>
- <span class="s1">'writeDisposition'</span><span class="p">:</span> <span class="n">write_disposition</span><span class="p">,</span>
- <span class="p">}</span>
- <span class="p">}</span>
- <span class="k">if</span> <span class="n">schema_fields</span><span class="p">:</span>
- <span class="n">configuration</span><span class="p">[</span><span class="s1">'load'</span><span class="p">][</span><span class="s1">'schema'</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
- <span class="s1">'fields'</span><span class="p">:</span> <span class="n">schema_fields</span>
- <span class="p">}</span>
-
- <span class="k">if</span> <span class="n">schema_update_options</span><span class="p">:</span>
- <span class="k">if</span> <span class="n">write_disposition</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">"WRITE_APPEND"</span><span class="p">,</span> <span class="s2">"WRITE_TRUNCATE"</span><span class="p">]:</span>
- <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
- <span class="s2">"schema_update_options is only "</span>
- <span class="s2">"allowed if write_disposition is "</span>
- <span class="s2">"'WRITE_APPEND' or 'WRITE_TRUNCATE'."</span>
- <span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
- <span class="s2">"Adding experimental "</span>
- <span class="s2">"'schemaUpdateOptions': </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">)</span>
- <span class="p">)</span>
- <span class="n">configuration</span><span class="p">[</span><span class="s1">'load'</span><span class="p">][</span><span class="s1">'schemaUpdateOptions'</span><span class="p">]</span> <span class="o">=</span> <span class="n">schema_update_options</span>
-
- <span class="k">if</span> <span class="n">max_bad_records</span><span class="p">:</span>
- <span class="n">configuration</span><span class="p">[</span><span class="s1">'load'</span><span class="p">][</span><span class="s1">'maxBadRecords'</span><span class="p">]</span> <span class="o">=</span> <span class="n">max_bad_records</span>
-
- <span class="c1"># if following fields are not specified in src_fmt_configs,</span>
- <span class="c1"># honor the top-level params for backward-compatibility</span>
- <span class="k">if</span> <span class="s1">'skipLeadingRows'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
- <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">'skipLeadingRows'</span><span class="p">]</span> <span class="o">=</span> <span class="n">skip_leading_rows</span>
- <span class="k">if</span> <span class="s1">'fieldDelimiter'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
- <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">'fieldDelimiter'</span><span class="p">]</span> <span class="o">=</span> <span class="n">field_delimiter</span>
- <span class="k">if</span> <span class="n">quote_character</span><span class="p">:</span>
- <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">'quote'</span><span class="p">]</span> <span class="o">=</span> <span class="n">quote_character</span>
- <span class="k">if</span> <span class="n">allow_quoted_newlines</span><span class="p">:</span>
- <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">'allowQuotedNewlines'</span><span class="p">]</span> <span class="o">=</span> <span class="n">allow_quoted_newlines</span>
-
- <span class="n">src_fmt_to_configs_mapping</span> <span class="o">=</span> <span class="p">{</span>
- <span class="s1">'CSV'</span><span class="p">:</span> <span class="p">[</span><span class="s1">'allowJaggedRows'</span><span class="p">,</span> <span class="s1">'allowQuotedNewlines'</span><span class="p">,</span> <span class="s1">'autodetect'</span><span class="p">,</span>
- <span class="s1">'fieldDelimiter'</span><span class="p">,</span> <span class="s1">'skipLeadingRows'</span><span class="p">,</span> <span class="s1">'ignoreUnknownValues'</span><span class="p">,</span>
- <span class="s1">'nullMarker'</span><span class="p">,</span> <span class="s1">'quote'</span><span class="p">],</span>
- <span class="s1">'DATASTORE_BACKUP'</span><span class="p">:</span> <span class="p">[</span><span class="s1">'projectionFields'</span><span class="p">],</span>
- <span class="s1">'NEWLINE_DELIMITED_JSON'</span><span class="p">:</span> <span class="p">[</span><span class="s1">'autodetect'</span><span class="p">,</span> <span class="s1">'ignoreUnknownValues'</span><span class="p">],</span>
- <span class="s1">'AVRO'</span><span class="p">:</span> <span class="p">[],</span>
- <span class="p">}</span>
- <span class="n">valid_configs</span> <span class="o">=</span> <span class="n">src_fmt_to_configs_mapping</span><span class="p">[</span><span class="n">source_format</span><span class="p">]</span>
- <span class="n">src_fmt_configs</span> <span class="o">=</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="o">.</span><span class="n">items</span><span class="p">()</span>
- <span class="k">if</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">valid_configs</span><span class="p">}</span>
- <span class="n">configuration</span><span class="p">[</span><span class="s1">'load'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">src_fmt_configs</span><span class="p">)</span>
-
- <span class="k">if</span> <span class="n">allow_jagged_rows</span><span class="p">:</span>
- <span class="n">configuration</span><span class="p">[</span><span class="s1">'load'</span><span class="p">][</span><span class="s1">'allowJaggedRows'</span><span class="p">]</span> <span class="o">=</span> <span class="n">allow_jagged_rows</span>
-
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_with_configuration</span><span class="p">(</span><span class="n">configuration</span><span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">run_with_configuration</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">configuration</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Executes a BigQuery SQL query. See here:</span>
-
-<span class="sd"> https://cloud.google.com/bigquery/docs/reference/v2/jobs</span>
-
-<span class="sd"> For more details about the configuration parameter.</span>
-
-<span class="sd"> :param configuration: The configuration parameter maps directly to</span>
-<span class="sd"> BigQuery's configuration field in the job object. See</span>
-<span class="sd"> https://cloud.google.com/bigquery/docs/reference/v2/jobs for</span>
-<span class="sd"> details.</span>
-<span class="sd"> """</span>
- <span class="n">jobs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span>
- <span class="n">job_data</span> <span class="o">=</span> <span class="p">{</span>
- <span class="s1">'configuration'</span><span class="p">:</span> <span class="n">configuration</span>
- <span class="p">}</span>
-
- <span class="c1"># Send query and wait for reply.</span>
- <span class="n">query_reply</span> <span class="o">=</span> <span class="n">jobs</span> \
- <span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">job_data</span><span class="p">)</span> \
- <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
- <span class="n">job_id</span> <span class="o">=</span> <span class="n">query_reply</span><span class="p">[</span><span class="s1">'jobReference'</span><span class="p">][</span><span class="s1">'jobId'</span><span class="p">]</span>
-
- <span class="c1"># Wait for query to finish.</span>
- <span class="n">keep_polling_job</span> <span class="o">=</span> <span class="kc">True</span>
- <span class="k">while</span> <span class="p">(</span><span class="n">keep_polling_job</span><span class="p">):</span>
- <span class="k">try</span><span class="p">:</span>
- <span class="n">job</span> <span class="o">=</span> <span class="n">jobs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">jobId</span><span class="o">=</span><span class="n">job_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
- <span class="k">if</span> <span class="p">(</span><span class="n">job</span><span class="p">[</span><span class="s1">'status'</span><span class="p">][</span><span class="s1">'state'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'DONE'</span><span class="p">):</span>
- <span class="n">keep_polling_job</span> <span class="o">=</span> <span class="kc">False</span>
- <span class="c1"># Check if job had errors.</span>
- <span class="k">if</span> <span class="s1">'errorResult'</span> <span class="ow">in</span> <span class="n">job</span><span class="p">[</span><span class="s1">'status'</span><span class="p">]:</span>
- <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
- <span class="s1">'BigQuery job failed. Final error was: </span><span class="si">{}</span><span class="s1">. The job was: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
- <span class="n">job</span><span class="p">[</span><span class="s1">'status'</span><span class="p">][</span><span class="s1">'errorResult'</span><span class="p">],</span> <span class="n">job</span>
- <span class="p">)</span>
- <span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Waiting for job to complete : </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job_id</span><span class="p">)</span>
- <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
-
- <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
- <span class="k">if</span> <span class="n">err</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span> <span class="ow">in</span> <span class="p">[</span><span class="mi">500</span><span class="p">,</span> <span class="mi">503</span><span class="p">]:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'</span><span class="si">%s</span><span class="s1">: Retryable error, waiting for job to complete: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">err</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span><span class="p">,</span> <span class="n">job_id</span><span class="p">)</span>
- <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
- <span class="s1">'BigQuery job status check failed. Final error was: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">err</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span><span class="p">)</span>
-
- <span class="k">return</span> <span class="n">job_id</span>
-
- <span class="k">def</span> <span class="nf">get_schema</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Get the schema for a given datset.table.</span>
-<span class="sd"> see https://cloud.google.com/bigquery/docs/reference/v2/tables#resource</span>
-
-<span class="sd"> :param dataset_id: the dataset ID of the requested table</span>
-<span class="sd"> :param table_id: the table ID of the requested table</span>
-<span class="sd"> :return: a table schema</span>
-<span class="sd"> """</span>
- <span class="n">tables_resource</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span> \
- <span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span> <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span><span class="p">)</span> \
- <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
- <span class="k">return</span> <span class="n">tables_resource</span><span class="p">[</span><span class="s1">'schema'</span><span class="p">]</span>
-
- <span class="k">def</span> <span class="nf">get_tabledata</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">,</span>
- <span class="n">max_results</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">page_token</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">start_index</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Get the data of a given dataset.table.</span>
-<span class="sd"> see https://cloud.google.com/bigquery/docs/reference/v2/tabledata/list</span>
-
-<span class="sd"> :param dataset_id: the dataset ID of the requested table.</span>
-<span class="sd"> :param table_id: the table ID of the requested table.</span>
-<span class="sd"> :param max_results: the maximum results to return.</span>
-<span class="sd"> :param page_token: page token, returned from a previous call,</span>
-<span class="sd"> identifying the result set.</span>
-<span class="sd"> :param start_index: zero based index of the starting row to read.</span>
-<span class="sd"> :return: map containing the requested rows.</span>
-<span class="sd"> """</span>
- <span class="n">optional_params</span> <span class="o">=</span> <span class="p">{}</span>
- <span class="k">if</span> <span class="n">max_results</span><span class="p">:</span>
- <span class="n">optional_params</span><span class="p">[</span><span class="s1">'maxResults'</span><span class="p">]</span> <span class="o">=</span> <span class="n">max_results</span>
- <span class="k">if</span> <span class="n">page_token</span><span class="p">:</span>
- <span class="n">optional_params</span><span class="p">[</span><span class="s1">'pageToken'</span><span class="p">]</span> <span class="o">=</span> <span class="n">page_token</span>
- <span class="k">if</span> <span class="n">start_index</span><span class="p">:</span>
- <span class="n">optional_params</span><span class="p">[</span><span class="s1">'startIndex'</span><span class="p">]</span> <span class="o">=</span> <span class="n">start_index</span>
- <span class="k">return</span> <span class="p">(</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tabledata</span><span class="p">()</span>
- <span class="o">.</span><span class="n">list</span><span class="p">(</span>
- <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
- <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span><span class="p">,</span> <span class="o">**</span><span class="n">optional_params</span><span class="p">)</span>
- <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
- <span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">run_table_delete</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">deletion_dataset_table</span><span class="p">,</span> <span class="n">ignore_if_missing</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Delete an existing table from the dataset;</span>
-<span class="sd"> If the table does not exist, return an error unless ignore_if_missing</span>
-<span class="sd"> is set to True.</span>
-
-<span class="sd"> :param deletion_dataset_table: A dotted</span>
-<span class="sd"> (<project>.|<project>:)<dataset>.<table> that indicates which table</span>
-<span class="sd"> will be deleted.</span>
-<span class="sd"> :type deletion_dataset_table: str</span>
-<span class="sd"> :param ignore_if_missing: if True, then return success even if the</span>
-<span class="sd"> requested table does not exist.</span>
-<span class="sd"> :type ignore_if_missing: boolean</span>
-<span class="sd"> :return:</span>
-<span class="sd"> """</span>
-
- <span class="k">assert</span> <span class="s1">'.'</span> <span class="ow">in</span> <span class="n">deletion_dataset_table</span><span class="p">,</span> <span class="p">(</span>
- <span class="s1">'Expected deletion_dataset_table in the format of '</span>
- <span class="s1">'<dataset>.<table>. Got: </span><span class="si">{}</span><span class="s1">'</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">deletion_dataset_table</span><span class="p">)</span>
- <span class="n">deletion_project</span><span class="p">,</span> <span class="n">deletion_dataset</span><span class="p">,</span> <span class="n">deletion_table</span> <span class="o">=</span> \
- <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">deletion_dataset_table</span><span class="p">,</span>
- <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">)</span>
-
- <span class="k">try</span><span class="p">:</span>
- <span class="n">tables_resource</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span> \
- <span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">deletion_project</span><span class="p">,</span>
- <span class="n">datasetId</span><span class="o">=</span><span class="n">deletion_dataset</span><span class="p">,</span>
- <span class="n">tableId</span><span class="o">=</span><span class="n">deletion_table</span><span class="p">)</span> \
- <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Deleted table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1">.'</span><span class="p">,</span>
- <span class="n">deletion_project</span><span class="p">,</span> <span class="n">deletion_dataset</span><span class="p">,</span> <span class="n">deletion_table</span><span class="p">)</span>
- <span class="k">except</span> <span class="n">HttpError</span><span class="p">:</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">ignore_if_missing</span><span class="p">:</span>
- <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
- <span class="s1">'Table deletion failed. Table does not exist.'</span><span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Table does not exist. Skipping.'</span><span class="p">)</span>
-
-
- <span class="k">def</span> <span class="nf">run_table_upsert</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_resource</span><span class="p">,</span> <span class="n">project_id</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> creates a new, empty table in the dataset;</span>
-<span class="sd"> If the table already exists, update the existing table.</span>
-<span class="sd"> Since BigQuery does not natively allow table upserts, this is not an</span>
-<span class="sd"> atomic operation.</span>
-
-<span class="sd"> :param dataset_id: the dataset to upsert the table into.</span>
-<span class="sd"> :type dataset_id: str</span>
-<span class="sd"> :param table_resource: a table resource. see</span>
-<span class="sd"> https://cloud.google.com/bigquery/docs/reference/v2/tables#resource</span>
-<span class="sd"> :type table_resource: dict</span>
-<span class="sd"> :param project_id: the project to upsert the table into. If None,</span>
-<span class="sd"> project will be self.project_id.</span>
-<span class="sd"> :return:</span>
-<span class="sd"> """</span>
- <span class="c1"># check to see if the table exists</span>
- <span class="n">table_id</span> <span class="o">=</span> <span class="n">table_resource</span><span class="p">[</span><span class="s1">'tableReference'</span><span class="p">][</span><span class="s1">'tableId'</span><span class="p">]</span>
- <span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span> <span class="k">if</span> <span class="n">project_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
- <span class="n">tables_list_resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">list</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
- <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
- <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
- <span class="k">for</span> <span class="n">table</span> <span class="ow">in</span> <span class="n">tables_list_resp</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'tables'</span><span class="p">,</span> <span class="p">[]):</span>
- <span class="k">if</span> <span class="n">table</span><span class="p">[</span><span class="s1">'tableReference'</span><span class="p">][</span><span class="s1">'tableId'</span><span class="p">]</span> <span class="o">==</span> <span class="n">table_id</span><span class="p">:</span>
- <span class="c1"># found the table, do update</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
- <span class="s1">'Table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> exists, updating.'</span><span class="p">,</span>
- <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span>
- <span class="p">)</span>
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
- <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
- <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span><span class="p">,</span>
- <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
- <span class="c1"># If there is a next page, we need to check the next page.</span>
- <span class="k">if</span> <span class="s1">'nextPageToken'</span> <span class="ow">in</span> <span class="n">tables_list_resp</span><span class="p">:</span>
- <span class="n">tables_list_resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span>\
- <span class="o">.</span><span class="n">list</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
- <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p
<TRUNCATED>
[07/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/s3_file_transform_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/s3_file_transform_operator.html b/_modules/airflow/operators/s3_file_transform_operator.html
index 8db7bc2..366400e 100644
--- a/_modules/airflow/operators/s3_file_transform_operator.html
+++ b/_modules/airflow/operators/s3_file_transform_operator.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,22 @@
<h1>Source code for airflow.operators.s3_file_transform_operator</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="kn">from</span> <span class="nn">tempfile</span> <span class="k">import</span> <span class="n">NamedTemporaryFile</span>
<span class="kn">import</span> <span class="nn">subprocess</span>
@@ -200,10 +207,13 @@
<span class="sd"> The locations of the source and the destination files in the local</span>
<span class="sd"> filesystem is provided as an first and second arguments to the</span>
<span class="sd"> transformation script. The transformation script is expected to read the</span>
-<span class="sd"> data from source , transform it and write the output to the local</span>
+<span class="sd"> data from source, transform it and write the output to the local</span>
<span class="sd"> destination file. The operator then takes over control and uploads the</span>
<span class="sd"> local destination file to S3.</span>
+<span class="sd"> S3 Select is also available to filter the source contents. Users can</span>
+<span class="sd"> omit the transformation script if S3 Select expression is specified.</span>
+
<span class="sd"> :param source_s3_key: The key to be retrieved from S3</span>
<span class="sd"> :type source_s3_key: str</span>
<span class="sd"> :param source_aws_conn_id: source s3 connection</span>
@@ -216,6 +226,8 @@
<span class="sd"> :type replace: bool</span>
<span class="sd"> :param transform_script: location of the executable transformation script</span>
<span class="sd"> :type transform_script: str</span>
+<span class="sd"> :param select_expression: S3 Select expression</span>
+<span class="sd"> :type select_expression: str</span>
<span class="sd"> """</span>
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'source_s3_key'</span><span class="p">,</span> <span class="s1">'dest_s3_key'</span><span class="p">)</span>
@@ -227,7 +239,8 @@
<span class="bp">self</span><span class="p">,</span>
<span class="n">source_s3_key</span><span class="p">,</span>
<span class="n">dest_s3_key</span><span class="p">,</span>
- <span class="n">transform_script</span><span class="p">,</span>
+ <span class="n">transform_script</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">select_expression</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">source_aws_conn_id</span><span class="o">=</span><span class="s1">'aws_default'</span><span class="p">,</span>
<span class="n">dest_aws_conn_id</span><span class="o">=</span><span class="s1">'aws_default'</span><span class="p">,</span>
<span class="n">replace</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
@@ -239,34 +252,54 @@
<span class="bp">self</span><span class="o">.</span><span class="n">dest_aws_conn_id</span> <span class="o">=</span> <span class="n">dest_aws_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">replace</span> <span class="o">=</span> <span class="n">replace</span>
<span class="bp">self</span><span class="o">.</span><span class="n">transform_script</span> <span class="o">=</span> <span class="n">transform_script</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">select_expression</span> <span class="o">=</span> <span class="n">select_expression</span>
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">transform_script</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">select_expression</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s2">"Either transform_script or select_expression must be specified"</span><span class="p">)</span>
+
<span class="n">source_s3</span> <span class="o">=</span> <span class="n">S3Hook</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">source_aws_conn_id</span><span class="p">)</span>
<span class="n">dest_s3</span> <span class="o">=</span> <span class="n">S3Hook</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dest_aws_conn_id</span><span class="p">)</span>
+
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Downloading source S3 file </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">source_s3_key</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">source_s3</span><span class="o">.</span><span class="n">check_for_key</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">source_s3_key</span><span class="p">):</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"The source key </span><span class="si">{0}</span><span class="s2"> does not exist"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">source_s3_key</span><span class="p">))</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s2">"The source key </span><span class="si">{0}</span><span class="s2"> does not exist"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">source_s3_key</span><span class="p">))</span>
<span class="n">source_s3_key_object</span> <span class="o">=</span> <span class="n">source_s3</span><span class="o">.</span><span class="n">get_key</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">source_s3_key</span><span class="p">)</span>
- <span class="k">with</span> <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="s2">"w"</span><span class="p">)</span> <span class="k">as</span> <span class="n">f_source</span><span class="p">,</span> <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="s2">"w"</span><span class="p">)</span> <span class="k">as</span> <span class="n">f_dest</span><span class="p">:</span>
+
+ <span class="k">with</span> <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="s2">"wb"</span><span class="p">)</span> <span class="k">as</span> <span class="n">f_source</span><span class="p">,</span> <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="s2">"wb"</span><span class="p">)</span> <span class="k">as</span> <span class="n">f_dest</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
<span class="s2">"Dumping S3 file </span><span class="si">%s</span><span class="s2"> contents to local file </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">source_s3_key</span><span class="p">,</span> <span class="n">f_source</span><span class="o">.</span><span class="n">name</span>
<span class="p">)</span>
- <span class="n">source_s3_key_object</span><span class="o">.</span><span class="n">get_contents_to_file</span><span class="p">(</span><span class="n">f_source</span><span class="p">)</span>
- <span class="n">f_source</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
- <span class="n">source_s3</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
- <span class="n">transform_script_process</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">Popen</span><span class="p">(</span>
- <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">transform_script</span><span class="p">,</span> <span class="n">f_source</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">f_dest</span><span class="o">.</span><span class="n">name</span><span class="p">],</span>
- <span class="n">stdout</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span> <span class="n">stderr</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">)</span>
- <span class="p">(</span><span class="n">transform_script_stdoutdata</span><span class="p">,</span> <span class="n">transform_script_stderrdata</span><span class="p">)</span> <span class="o">=</span> <span class="n">transform_script_process</span><span class="o">.</span><span class="n">communicate</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Transform script stdout </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">transform_script_stdoutdata</span><span class="p">)</span>
- <span class="k">if</span> <span class="n">transform_script_process</span><span class="o">.</span><span class="n">returncode</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Transform script failed </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">transform_script_stderrdata</span><span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
- <span class="s2">"Transform script successful. Output temporarily located at </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
- <span class="n">f_dest</span><span class="o">.</span><span class="n">name</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">select_expression</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">content</span> <span class="o">=</span> <span class="n">source_s3</span><span class="o">.</span><span class="n">select_key</span><span class="p">(</span>
+ <span class="n">key</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">source_s3_key</span><span class="p">,</span>
+ <span class="n">expression</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">select_expression</span>
<span class="p">)</span>
+ <span class="n">f_source</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">content</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s2">"utf-8"</span><span class="p">))</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">source_s3_key_object</span><span class="o">.</span><span class="n">download_fileobj</span><span class="p">(</span><span class="n">Fileobj</span><span class="o">=</span><span class="n">f_source</span><span class="p">)</span>
+ <span class="n">f_source</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">transform_script</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">transform_script_process</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">Popen</span><span class="p">(</span>
+ <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">transform_script</span><span class="p">,</span> <span class="n">f_source</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">f_dest</span><span class="o">.</span><span class="n">name</span><span class="p">],</span>
+ <span class="n">stdout</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span> <span class="n">stderr</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span> <span class="n">close_fds</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="p">(</span><span class="n">transform_script_stdoutdata</span><span class="p">,</span> <span class="n">transform_script_stderrdata</span><span class="p">)</span> <span class="o">=</span> \
+ <span class="n">transform_script_process</span><span class="o">.</span><span class="n">communicate</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Transform script stdout </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">transform_script_stdoutdata</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">transform_script_process</span><span class="o">.</span><span class="n">returncode</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s2">"Transform script failed </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">transform_script_stderrdata</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s2">"Transform script successful. Output temporarily located at </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
+ <span class="n">f_dest</span><span class="o">.</span><span class="n">name</span>
+ <span class="p">)</span>
+
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Uploading transformed file to S3"</span><span class="p">)</span>
<span class="n">f_dest</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
<span class="n">dest_s3</span><span class="o">.</span><span class="n">load_file</span><span class="p">(</span>
@@ -274,8 +307,7 @@
<span class="n">key</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dest_s3_key</span><span class="p">,</span>
<span class="n">replace</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">replace</span>
<span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Upload successful"</span><span class="p">)</span>
- <span class="n">dest_s3</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">close</span><span class="p">()</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Upload successful"</span><span class="p">)</span></div>
</pre></div>
</div>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/s3_to_hive_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/s3_to_hive_operator.html b/_modules/airflow/operators/s3_to_hive_operator.html
deleted file mode 100644
index be79923..0000000
--- a/_modules/airflow/operators/s3_to_hive_operator.html
+++ /dev/null
@@ -1,482 +0,0 @@
-
-
-<!DOCTYPE html>
-<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
-<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
-<head>
- <meta charset="utf-8">
-
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
-
- <title>airflow.operators.s3_to_hive_operator — Airflow Documentation</title>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
- <link rel="up" title="Module code" href="../../index.html"/>
-
-
- <script src="../../../_static/js/modernizr.min.js"></script>
-
-</head>
-
-<body class="wy-body-for-nav" role="document">
-
-
- <div class="wy-grid-for-nav">
-
-
- <nav data-toggle="wy-nav-shift" class="wy-nav-side">
- <div class="wy-side-scroll">
- <div class="wy-side-nav-search">
-
-
-
- <a href="../../../index.html" class="icon icon-home"> Airflow
-
-
-
- </a>
-
-
-
-
-
-
-
-<div role="search">
- <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
- <input type="text" name="q" placeholder="Search docs" />
- <input type="hidden" name="check_keywords" value="yes" />
- <input type="hidden" name="area" value="default" />
- </form>
-</div>
-
-
- </div>
-
- <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
-
-
-
-
-
-
- <ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../configuration.html">Configuration</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
-</ul>
-
-
-
- </div>
- </div>
- </nav>
-
- <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
-
-
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
-
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="../../../index.html">Airflow</a>
-
- </nav>
-
-
-
- <div class="wy-nav-content">
- <div class="rst-content">
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-<div role="navigation" aria-label="breadcrumbs navigation">
-
- <ul class="wy-breadcrumbs">
-
- <li><a href="../../../index.html">Docs</a> »</li>
-
- <li><a href="../../index.html">Module code</a> »</li>
-
- <li>airflow.operators.s3_to_hive_operator</li>
-
-
- <li class="wy-breadcrumbs-aside">
-
-
-
- </li>
-
- </ul>
-
-
- <hr/>
-</div>
- <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
- <div itemprop="articleBody">
-
- <h1>Source code for airflow.operators.s3_to_hive_operator</h1><div class="highlight"><pre>
-<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
-<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
-
-<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">next</span>
-<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">zip</span>
-<span class="kn">from</span> <span class="nn">tempfile</span> <span class="k">import</span> <span class="n">NamedTemporaryFile</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.file</span> <span class="k">import</span> <span class="n">TemporaryDirectory</span>
-<span class="kn">import</span> <span class="nn">gzip</span>
-<span class="kn">import</span> <span class="nn">bz2</span>
-<span class="kn">import</span> <span class="nn">tempfile</span>
-<span class="kn">import</span> <span class="nn">os</span>
-
-<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
-<span class="kn">from</span> <span class="nn">airflow.hooks.S3_hook</span> <span class="k">import</span> <span class="n">S3Hook</span>
-<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="k">import</span> <span class="n">HiveCliHook</span>
-<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.compression</span> <span class="k">import</span> <span class="n">uncompress_file</span>
-
-
-<div class="viewcode-block" id="S3ToHiveTransfer"><a class="viewcode-back" href="../../../integration.html#airflow.operators.s3_to_hive_operator.S3ToHiveTransfer">[docs]</a><span class="k">class</span> <span class="nc">S3ToHiveTransfer</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Moves data from S3 to Hive. The operator downloads a file from S3,</span>
-<span class="sd"> stores the file locally before loading it into a Hive table.</span>
-<span class="sd"> If the ``create`` or ``recreate`` arguments are set to ``True``,</span>
-<span class="sd"> a ``CREATE TABLE`` and ``DROP TABLE`` statements are generated.</span>
-<span class="sd"> Hive data types are inferred from the cursor's metadata from.</span>
-
-<span class="sd"> Note that the table generated in Hive uses ``STORED AS textfile``</span>
-<span class="sd"> which isn't the most efficient serialization format. If a</span>
-<span class="sd"> large amount of data is loaded and/or if the tables gets</span>
-<span class="sd"> queried considerably, you may want to use this operator only to</span>
-<span class="sd"> stage the data into a temporary table before loading it into its</span>
-<span class="sd"> final destination using a ``HiveOperator``.</span>
-
-<span class="sd"> :param s3_key: The key to be retrieved from S3</span>
-<span class="sd"> :type s3_key: str</span>
-<span class="sd"> :param field_dict: A dictionary of the fields name in the file</span>
-<span class="sd"> as keys and their Hive types as values</span>
-<span class="sd"> :type field_dict: dict</span>
-<span class="sd"> :param hive_table: target Hive table, use dot notation to target a</span>
-<span class="sd"> specific database</span>
-<span class="sd"> :type hive_table: str</span>
-<span class="sd"> :param create: whether to create the table if it doesn't exist</span>
-<span class="sd"> :type create: bool</span>
-<span class="sd"> :param recreate: whether to drop and recreate the table at every</span>
-<span class="sd"> execution</span>
-<span class="sd"> :type recreate: bool</span>
-<span class="sd"> :param partition: target partition as a dict of partition columns</span>
-<span class="sd"> and values</span>
-<span class="sd"> :type partition: dict</span>
-<span class="sd"> :param headers: whether the file contains column names on the first</span>
-<span class="sd"> line</span>
-<span class="sd"> :type headers: bool</span>
-<span class="sd"> :param check_headers: whether the column names on the first line should be</span>
-<span class="sd"> checked against the keys of field_dict</span>
-<span class="sd"> :type check_headers: bool</span>
-<span class="sd"> :param wildcard_match: whether the s3_key should be interpreted as a Unix</span>
-<span class="sd"> wildcard pattern</span>
-<span class="sd"> :type wildcard_match: bool</span>
-<span class="sd"> :param delimiter: field delimiter in the file</span>
-<span class="sd"> :type delimiter: str</span>
-<span class="sd"> :param aws_conn_id: source s3 connection</span>
-<span class="sd"> :type aws_conn_id: str</span>
-<span class="sd"> :param hive_cli_conn_id: destination hive connection</span>
-<span class="sd"> :type hive_cli_conn_id: str</span>
-<span class="sd"> :param input_compressed: Boolean to determine if file decompression is</span>
-<span class="sd"> required to process headers</span>
-<span class="sd"> :type input_compressed: bool</span>
-<span class="sd"> :param tblproperties: TBLPROPERTIES of the hive table being created</span>
-<span class="sd"> :type tblproperties: dict</span>
-<span class="sd"> """</span>
-
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'s3_key'</span><span class="p">,</span> <span class="s1">'partition'</span><span class="p">,</span> <span class="s1">'hive_table'</span><span class="p">)</span>
- <span class="n">template_ext</span> <span class="o">=</span> <span class="p">()</span>
- <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#a0e08c'</span>
-
- <span class="nd">@apply_defaults</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
- <span class="bp">self</span><span class="p">,</span>
- <span class="n">s3_key</span><span class="p">,</span>
- <span class="n">field_dict</span><span class="p">,</span>
- <span class="n">hive_table</span><span class="p">,</span>
- <span class="n">delimiter</span><span class="o">=</span><span class="s1">','</span><span class="p">,</span>
- <span class="n">create</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
- <span class="n">recreate</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">partition</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">headers</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">check_headers</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">wildcard_match</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">aws_conn_id</span><span class="o">=</span><span class="s1">'aws_default'</span><span class="p">,</span>
- <span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="s1">'hive_cli_default'</span><span class="p">,</span>
- <span class="n">input_compressed</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">tblproperties</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">S3ToHiveTransfer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">s3_key</span> <span class="o">=</span> <span class="n">s3_key</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">field_dict</span> <span class="o">=</span> <span class="n">field_dict</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">hive_table</span> <span class="o">=</span> <span class="n">hive_table</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">delimiter</span> <span class="o">=</span> <span class="n">delimiter</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">create</span> <span class="o">=</span> <span class="n">create</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">recreate</span> <span class="o">=</span> <span class="n">recreate</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">partition</span> <span class="o">=</span> <span class="n">partition</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">headers</span> <span class="o">=</span> <span class="n">headers</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">check_headers</span> <span class="o">=</span> <span class="n">check_headers</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">wildcard_match</span> <span class="o">=</span> <span class="n">wildcard_match</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_conn_id</span> <span class="o">=</span> <span class="n">hive_cli_conn_id</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span> <span class="o">=</span> <span class="n">aws_conn_id</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">input_compressed</span> <span class="o">=</span> <span class="n">input_compressed</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">tblproperties</span> <span class="o">=</span> <span class="n">tblproperties</span>
-
- <span class="k">if</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">check_headers</span> <span class="ow">and</span>
- <span class="ow">not</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">field_dict</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">headers</span><span class="p">)):</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"To check_headers provide "</span> <span class="o">+</span>
- <span class="s2">"field_dict and headers"</span><span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
- <span class="c1"># Downloading file from S3</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">s3</span> <span class="o">=</span> <span class="n">S3Hook</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">hive</span> <span class="o">=</span> <span class="n">HiveCliHook</span><span class="p">(</span><span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_conn_id</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Downloading S3 file"</span><span class="p">)</span>
-
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">wildcard_match</span><span class="p">:</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">s3</span><span class="o">.</span><span class="n">check_for_wildcard_key</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_key</span><span class="p">):</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"No key matches </span><span class="si">{0}</span><span class="s2">"</span>
- <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_key</span><span class="p">))</span>
- <span class="n">s3_key_object</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">s3</span><span class="o">.</span><span class="n">get_wildcard_key</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_key</span><span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">s3</span><span class="o">.</span><span class="n">check_for_key</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_key</span><span class="p">):</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
- <span class="s2">"The key </span><span class="si">{0}</span><span class="s2"> does not exists"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_key</span><span class="p">))</span>
- <span class="n">s3_key_object</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">s3</span><span class="o">.</span><span class="n">get_key</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_key</span><span class="p">)</span>
- <span class="n">root</span><span class="p">,</span> <span class="n">file_ext</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">splitext</span><span class="p">(</span><span class="n">s3_key_object</span><span class="o">.</span><span class="n">key</span><span class="p">)</span>
- <span class="k">with</span> <span class="n">TemporaryDirectory</span><span class="p">(</span><span class="n">prefix</span><span class="o">=</span><span class="s1">'tmps32hive_'</span><span class="p">)</span> <span class="k">as</span> <span class="n">tmp_dir</span><span class="p">,</span>\
- <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="n">mode</span><span class="o">=</span><span class="s2">"wb"</span><span class="p">,</span>
- <span class="nb">dir</span><span class="o">=</span><span class="n">tmp_dir</span><span class="p">,</span>
- <span class="n">suffix</span><span class="o">=</span><span class="n">file_ext</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Dumping S3 key </span><span class="si">{0}</span><span class="s2"> contents to local file </span><span class="si">{1}</span><span class="s2">"</span>
- <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">s3_key_object</span><span class="o">.</span><span class="n">key</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="p">))</span>
- <span class="n">s3_key_object</span><span class="o">.</span><span class="n">download_fileobj</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
- <span class="n">f</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">headers</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Loading file </span><span class="si">%s</span><span class="s2"> into Hive"</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">hive</span><span class="o">.</span><span class="n">load_file</span><span class="p">(</span>
- <span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="p">,</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">hive_table</span><span class="p">,</span>
- <span class="n">field_dict</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">field_dict</span><span class="p">,</span>
- <span class="n">create</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">create</span><span class="p">,</span>
- <span class="n">partition</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">partition</span><span class="p">,</span>
- <span class="n">delimiter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delimiter</span><span class="p">,</span>
- <span class="n">recreate</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">recreate</span><span class="p">,</span>
- <span class="n">tblproperties</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">tblproperties</span><span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="c1"># Decompressing file</span>
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">input_compressed</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Uncompressing file </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
- <span class="n">fn_uncompressed</span> <span class="o">=</span> <span class="n">uncompress_file</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="p">,</span>
- <span class="n">file_ext</span><span class="p">,</span>
- <span class="n">tmp_dir</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Uncompressed to </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">fn_uncompressed</span><span class="p">)</span>
- <span class="c1"># uncompressed file available now so deleting</span>
- <span class="c1"># compressed file to save disk space</span>
- <span class="n">f</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">fn_uncompressed</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">name</span>
-
- <span class="c1"># Testing if header matches field_dict</span>
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">check_headers</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Matching file header against field_dict"</span><span class="p">)</span>
- <span class="n">header_list</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_top_row_as_list</span><span class="p">(</span><span class="n">fn_uncompressed</span><span class="p">)</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_match_headers</span><span class="p">(</span><span class="n">header_list</span><span class="p">):</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Header check failed"</span><span class="p">)</span>
-
- <span class="c1"># Deleting top header row</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Removing header from file </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">fn_uncompressed</span><span class="p">)</span>
- <span class="n">headless_file</span> <span class="o">=</span> <span class="p">(</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">_delete_top_row_and_compress</span><span class="p">(</span><span class="n">fn_uncompressed</span><span class="p">,</span>
- <span class="n">file_ext</span><span class="p">,</span>
- <span class="n">tmp_dir</span><span class="p">))</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Headless file </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">headless_file</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Loading file </span><span class="si">%s</span><span class="s2"> into Hive"</span><span class="p">,</span> <span class="n">headless_file</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">hive</span><span class="o">.</span><span class="n">load_file</span><span class="p">(</span><span class="n">headless_file</span><span class="p">,</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">hive_table</span><span class="p">,</span>
- <span class="n">field_dict</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">field_dict</span><span class="p">,</span>
- <span class="n">create</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">create</span><span class="p">,</span>
- <span class="n">partition</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">partition</span><span class="p">,</span>
- <span class="n">delimiter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delimiter</span><span class="p">,</span>
- <span class="n">recreate</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">recreate</span><span class="p">,</span>
- <span class="n">tblproperties</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">tblproperties</span><span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">_get_top_row_as_list</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">file_name</span><span class="p">):</span>
- <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">file_name</span><span class="p">,</span> <span class="s1">'rt'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
- <span class="n">header_line</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">readline</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
- <span class="n">header_list</span> <span class="o">=</span> <span class="n">header_line</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">delimiter</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">header_list</span>
-
- <span class="k">def</span> <span class="nf">_match_headers</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">header_list</span><span class="p">):</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">header_list</span><span class="p">:</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Unable to retrieve header row from file"</span><span class="p">)</span>
- <span class="n">field_names</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">field_dict</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span>
- <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">field_names</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="n">header_list</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Headers count mismatch"</span>
- <span class="s2">"File headers:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{header_list}</span><span class="se">\n</span><span class="s2">"</span>
- <span class="s2">"Field names: </span><span class="se">\n</span><span class="s2"> </span><span class="si">{field_names}</span><span class="se">\n</span><span class="s2">"</span>
- <span class="s2">""</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
- <span class="k">return</span> <span class="kc">False</span>
- <span class="n">test_field_match</span> <span class="o">=</span> <span class="p">[</span><span class="n">h1</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="n">h2</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
- <span class="k">for</span> <span class="n">h1</span><span class="p">,</span> <span class="n">h2</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">header_list</span><span class="p">,</span> <span class="n">field_names</span><span class="p">)]</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="nb">all</span><span class="p">(</span><span class="n">test_field_match</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Headers do not match field names"</span>
- <span class="s2">"File headers:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{header_list}</span><span class="se">\n</span><span class="s2">"</span>
- <span class="s2">"Field names: </span><span class="se">\n</span><span class="s2"> </span><span class="si">{field_names}</span><span class="se">\n</span><span class="s2">"</span>
- <span class="s2">""</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
- <span class="k">return</span> <span class="kc">False</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="k">return</span> <span class="kc">True</span>
-
- <span class="k">def</span> <span class="nf">_delete_top_row_and_compress</span><span class="p">(</span>
- <span class="bp">self</span><span class="p">,</span>
- <span class="n">input_file_name</span><span class="p">,</span>
- <span class="n">output_file_ext</span><span class="p">,</span>
- <span class="n">dest_dir</span><span class="p">):</span>
- <span class="c1"># When output_file_ext is not defined, file is not compressed</span>
- <span class="n">open_fn</span> <span class="o">=</span> <span class="nb">open</span>
- <span class="k">if</span> <span class="n">output_file_ext</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'.gz'</span><span class="p">:</span>
- <span class="n">open_fn</span> <span class="o">=</span> <span class="n">gzip</span><span class="o">.</span><span class="n">GzipFile</span>
- <span class="k">elif</span> <span class="n">output_file_ext</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'.bz2'</span><span class="p">:</span>
- <span class="n">open_fn</span> <span class="o">=</span> <span class="n">bz2</span><span class="o">.</span><span class="n">BZ2File</span>
-
- <span class="n">os_fh_output</span><span class="p">,</span> <span class="n">fn_output</span> <span class="o">=</span> \
- <span class="n">tempfile</span><span class="o">.</span><span class="n">mkstemp</span><span class="p">(</span><span class="n">suffix</span><span class="o">=</span><span class="n">output_file_ext</span><span class="p">,</span> <span class="nb">dir</span><span class="o">=</span><span class="n">dest_dir</span><span class="p">)</span>
- <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">input_file_name</span><span class="p">,</span> <span class="s1">'rb'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f_in</span><span class="p">,</span>\
- <span class="n">open_fn</span><span class="p">(</span><span class="n">fn_output</span><span class="p">,</span> <span class="s1">'wb'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f_out</span><span class="p">:</span>
- <span class="n">f_in</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
- <span class="nb">next</span><span class="p">(</span><span class="n">f_in</span><span class="p">)</span>
- <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">f_in</span><span class="p">:</span>
- <span class="n">f_out</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">line</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">fn_output</span></div>
-</pre></div>
-
- </div>
- <div class="articleComments">
-
- </div>
- </div>
- <footer>
-
-
- <hr/>
-
- <div role="contentinfo">
- <p>
-
- </p>
- </div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
-
-</footer>
-
- </div>
- </div>
-
- </section>
-
- </div>
-
-
-
-
-
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../',
- VERSION:'',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../_static/doctools.js"></script>
-
-
-
-
-
- <script type="text/javascript" src="../../../_static/js/theme.js"></script>
-
-
-
-
- <script type="text/javascript">
- jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
- });
- </script>
-
-
-</body>
-</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/s3_to_redshift_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/s3_to_redshift_operator.html b/_modules/airflow/operators/s3_to_redshift_operator.html
new file mode 100644
index 0000000..e83cdfd
--- /dev/null
+++ b/_modules/airflow/operators/s3_to_redshift_operator.html
@@ -0,0 +1,330 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.operators.s3_to_redshift_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.operators.s3_to_redshift_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.operators.s3_to_redshift_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.postgres_hook</span> <span class="k">import</span> <span class="n">PostgresHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.S3_hook</span> <span class="k">import</span> <span class="n">S3Hook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="S3ToRedshiftTransfer"><a class="viewcode-back" href="../../../integration.html#airflow.operators.s3_to_redshift_operator.S3ToRedshiftTransfer">[docs]</a><span class="k">class</span> <span class="nc">S3ToRedshiftTransfer</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Executes an COPY command to load files from s3 to Redshift</span>
+
+<span class="sd"> :param schema: reference to a specific schema in redshift database</span>
+<span class="sd"> :type schema: string</span>
+<span class="sd"> :param table: reference to a specific table in redshift database</span>
+<span class="sd"> :type table: string</span>
+<span class="sd"> :param s3_bucket: reference to a specific S3 bucket</span>
+<span class="sd"> :type s3_bucket: string</span>
+<span class="sd"> :param s3_key: reference to a specific S3 key</span>
+<span class="sd"> :type s3_key: string</span>
+<span class="sd"> :param redshift_conn_id: reference to a specific redshift database</span>
+<span class="sd"> :type redshift_conn_id: string</span>
+<span class="sd"> :param aws_conn_id: reference to a specific S3 connection</span>
+<span class="sd"> :type aws_conn_id: string</span>
+<span class="sd"> :param copy_options: reference to a list of COPY options</span>
+<span class="sd"> :type copy_options: list</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">()</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">()</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#ededed'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">schema</span><span class="p">,</span>
+ <span class="n">table</span><span class="p">,</span>
+ <span class="n">s3_bucket</span><span class="p">,</span>
+ <span class="n">s3_key</span><span class="p">,</span>
+ <span class="n">redshift_conn_id</span><span class="o">=</span><span class="s1">'redshift_default'</span><span class="p">,</span>
+ <span class="n">aws_conn_id</span><span class="o">=</span><span class="s1">'aws_default'</span><span class="p">,</span>
+ <span class="n">copy_options</span><span class="o">=</span><span class="nb">tuple</span><span class="p">(),</span>
+ <span class="n">autocommit</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">S3ToRedshiftTransfer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="o">=</span> <span class="n">schema</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">table</span> <span class="o">=</span> <span class="n">table</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">s3_bucket</span> <span class="o">=</span> <span class="n">s3_bucket</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">s3_key</span> <span class="o">=</span> <span class="n">s3_key</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">redshift_conn_id</span> <span class="o">=</span> <span class="n">redshift_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span> <span class="o">=</span> <span class="n">aws_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">copy_options</span> <span class="o">=</span> <span class="n">copy_options</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">autocommit</span> <span class="o">=</span> <span class="n">autocommit</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span> <span class="o">=</span> <span class="n">parameters</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">PostgresHook</span><span class="p">(</span><span class="n">postgres_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">redshift_conn_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">s3</span> <span class="o">=</span> <span class="n">S3Hook</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span>
+ <span class="n">credentials</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">s3</span><span class="o">.</span><span class="n">get_credentials</span><span class="p">()</span>
+ <span class="n">copy_options</span> <span class="o">=</span> <span class="s1">'</span><span class="se">\n\t\t\t</span><span class="s1">'</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">copy_options</span><span class="p">)</span>
+
+ <span class="n">copy_query</span> <span class="o">=</span> <span class="s2">"""</span>
+<span class="s2"> COPY </span><span class="si">{schema}</span><span class="s2">.</span><span class="si">{table}</span><span class="s2"></span>
+<span class="s2"> FROM 's3://</span><span class="si">{s3_bucket}</span><span class="s2">/</span><span class="si">{s3_key}</span><span class="s2">/</span><span class="si">{table}</span><span class="s2">'</span>
+<span class="s2"> with credentials</span>
+<span class="s2"> 'aws_access_key_id=</span><span class="si">{access_key}</span><span class="s2">;aws_secret_access_key=</span><span class="si">{secret_key}</span><span class="s2">'</span>
+<span class="s2"> </span><span class="si">{copy_options}</span><span class="s2">;</span>
+<span class="s2"> """</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">schema</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span>
+ <span class="n">table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="p">,</span>
+ <span class="n">s3_bucket</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_bucket</span><span class="p">,</span>
+ <span class="n">s3_key</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_key</span><span class="p">,</span>
+ <span class="n">access_key</span><span class="o">=</span><span class="n">credentials</span><span class="o">.</span><span class="n">access_key</span><span class="p">,</span>
+ <span class="n">secret_key</span><span class="o">=</span><span class="n">credentials</span><span class="o">.</span><span class="n">secret_key</span><span class="p">,</span>
+ <span class="n">copy_options</span><span class="o">=</span><span class="n">copy_options</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing COPY command...'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">copy_query</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">autocommit</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"COPY command complete..."</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[16/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/hooks/hdfs_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/hooks/hdfs_hook.html b/_modules/airflow/hooks/hdfs_hook.html
new file mode 100644
index 0000000..664c043
--- /dev/null
+++ b/_modules/airflow/hooks/hdfs_hook.html
@@ -0,0 +1,335 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.hooks.hdfs_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.hooks.hdfs_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.hooks.hdfs_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">configuration</span>
+
+<span class="k">try</span><span class="p">:</span>
+ <span class="n">snakebite_imported</span> <span class="o">=</span> <span class="kc">True</span>
+ <span class="kn">from</span> <span class="nn">snakebite.client</span> <span class="k">import</span> <span class="n">Client</span><span class="p">,</span> <span class="n">HAClient</span><span class="p">,</span> <span class="n">Namenode</span><span class="p">,</span> <span class="n">AutoConfigClient</span>
+<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
+ <span class="n">snakebite_imported</span> <span class="o">=</span> <span class="kc">False</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+
+
+<span class="k">class</span> <span class="nc">HDFSHookException</span><span class="p">(</span><span class="n">AirflowException</span><span class="p">):</span>
+ <span class="k">pass</span>
+
+
+<div class="viewcode-block" id="HDFSHook"><a class="viewcode-back" href="../../../code.html#airflow.hooks.hdfs_hook.HDFSHook">[docs]</a><span class="k">class</span> <span class="nc">HDFSHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with HDFS. This class is a wrapper around the snakebite library.</span>
+
+<span class="sd"> :param hdfs_conn_id: Connection id to fetch connection info</span>
+<span class="sd"> :type conn_id: string</span>
+<span class="sd"> :param proxy_user: effective user for HDFS operations</span>
+<span class="sd"> :type proxy_user: string</span>
+<span class="sd"> :param autoconfig: use snakebite's automatically configured client</span>
+<span class="sd"> :type autoconfig: bool</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hdfs_conn_id</span><span class="o">=</span><span class="s1">'hdfs_default'</span><span class="p">,</span> <span class="n">proxy_user</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">autoconfig</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">snakebite_imported</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ImportError</span><span class="p">(</span>
+ <span class="s1">'This HDFSHook implementation requires snakebite, but '</span>
+ <span class="s1">'snakebite is not compatible with Python 3 '</span>
+ <span class="s1">'(as of August 2015). Please use Python 2 if you require '</span>
+ <span class="s1">'this hook -- or help by submitting a PR!'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span> <span class="o">=</span> <span class="n">hdfs_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">proxy_user</span> <span class="o">=</span> <span class="n">proxy_user</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">autoconfig</span> <span class="o">=</span> <span class="n">autoconfig</span>
+
+<div class="viewcode-block" id="HDFSHook.get_conn"><a class="viewcode-back" href="../../../code.html#airflow.hooks.hdfs_hook.HDFSHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns a snakebite HDFSClient object.</span>
+<span class="sd"> """</span>
+ <span class="c1"># When using HAClient, proxy_user must be the same, so is ok to always</span>
+ <span class="c1"># take the first.</span>
+ <span class="n">effective_user</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">proxy_user</span>
+ <span class="n">autoconfig</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">autoconfig</span>
+ <span class="n">use_sasl</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'security'</span><span class="p">)</span> <span class="o">==</span> <span class="s1">'kerberos'</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">connections</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connections</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">effective_user</span><span class="p">:</span>
+ <span class="n">effective_user</span> <span class="o">=</span> <span class="n">connections</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">login</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">autoconfig</span><span class="p">:</span>
+ <span class="n">autoconfig</span> <span class="o">=</span> <span class="n">connections</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'autoconfig'</span><span class="p">,</span>
+ <span class="kc">False</span><span class="p">)</span>
+ <span class="n">hdfs_namenode_principal</span> <span class="o">=</span> <span class="n">connections</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="s1">'hdfs_namenode_principal'</span><span class="p">)</span>
+ <span class="k">except</span> <span class="n">AirflowException</span><span class="p">:</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">autoconfig</span><span class="p">:</span>
+ <span class="k">raise</span>
+
+ <span class="k">if</span> <span class="n">autoconfig</span><span class="p">:</span>
+ <span class="c1"># will read config info from $HADOOP_HOME conf files</span>
+ <span class="n">client</span> <span class="o">=</span> <span class="n">AutoConfigClient</span><span class="p">(</span><span class="n">effective_user</span><span class="o">=</span><span class="n">effective_user</span><span class="p">,</span>
+ <span class="n">use_sasl</span><span class="o">=</span><span class="n">use_sasl</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="nb">len</span><span class="p">(</span><span class="n">connections</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
+ <span class="n">client</span> <span class="o">=</span> <span class="n">Client</span><span class="p">(</span><span class="n">connections</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> <span class="n">connections</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">port</span><span class="p">,</span>
+ <span class="n">effective_user</span><span class="o">=</span><span class="n">effective_user</span><span class="p">,</span> <span class="n">use_sasl</span><span class="o">=</span><span class="n">use_sasl</span><span class="p">,</span>
+ <span class="n">hdfs_namenode_principal</span><span class="o">=</span><span class="n">hdfs_namenode_principal</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="nb">len</span><span class="p">(</span><span class="n">connections</span><span class="p">)</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
+ <span class="n">nn</span> <span class="o">=</span> <span class="p">[</span><span class="n">Namenode</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">)</span> <span class="k">for</span> <span class="n">conn</span> <span class="ow">in</span> <span class="n">connections</span><span class="p">]</span>
+ <span class="n">client</span> <span class="o">=</span> <span class="n">HAClient</span><span class="p">(</span><span class="n">nn</span><span class="p">,</span> <span class="n">effective_user</span><span class="o">=</span><span class="n">effective_user</span><span class="p">,</span>
+ <span class="n">use_sasl</span><span class="o">=</span><span class="n">use_sasl</span><span class="p">,</span>
+ <span class="n">hdfs_namenode_principal</span><span class="o">=</span><span class="n">hdfs_namenode_principal</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">HDFSHookException</span><span class="p">(</span><span class="s2">"conn_id doesn't exist in the repository "</span>
+ <span class="s2">"and autoconfig is not specified"</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">client</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/hooks/http_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/hooks/http_hook.html b/_modules/airflow/hooks/http_hook.html
new file mode 100644
index 0000000..0df6ddc
--- /dev/null
+++ b/_modules/airflow/hooks/http_hook.html
@@ -0,0 +1,358 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.hooks.http_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.hooks.http_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.hooks.http_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
+
+<span class="kn">import</span> <span class="nn">requests</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+
+
+<div class="viewcode-block" id="HttpHook"><a class="viewcode-back" href="../../../code.html#airflow.hooks.http_hook.HttpHook">[docs]</a><span class="k">class</span> <span class="nc">HttpHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with HTTP servers.</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">method</span><span class="o">=</span><span class="s1">'POST'</span><span class="p">,</span> <span class="n">http_conn_id</span><span class="o">=</span><span class="s1">'http_default'</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">http_conn_id</span> <span class="o">=</span> <span class="n">http_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">=</span> <span class="n">method</span>
+
+ <span class="c1"># headers is required to make it required</span>
+<div class="viewcode-block" id="HttpHook.get_conn"><a class="viewcode-back" href="../../../code.html#airflow.hooks.http_hook.HttpHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">headers</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns http session for use with requests</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">http_conn_id</span><span class="p">)</span>
+ <span class="n">session</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span>
+
+ <span class="k">if</span> <span class="s2">"://"</span> <span class="ow">in</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="c1"># schema defaults to HTTP</span>
+ <span class="n">schema</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">schema</span> <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">schema</span> <span class="k">else</span> <span class="s2">"http"</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span> <span class="o">=</span> <span class="n">schema</span> <span class="o">+</span> <span class="s2">"://"</span> <span class="o">+</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span>
+
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span> <span class="o">+</span> <span class="s2">":"</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">)</span> <span class="o">+</span> <span class="s2">"/"</span>
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">:</span>
+ <span class="n">session</span><span class="o">.</span><span class="n">auth</span> <span class="o">=</span> <span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">headers</span><span class="p">:</span>
+ <span class="n">session</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">headers</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">session</span></div>
+
+<div class="viewcode-block" id="HttpHook.run"><a class="viewcode-back" href="../../../code.html#airflow.hooks.http_hook.HttpHook.run">[docs]</a> <span class="k">def</span> <span class="nf">run</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">endpoint</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">extra_options</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Performs the request</span>
+<span class="sd"> """</span>
+ <span class="n">extra_options</span> <span class="o">=</span> <span class="n">extra_options</span> <span class="ow">or</span> <span class="p">{}</span>
+
+ <span class="n">session</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">(</span><span class="n">headers</span><span class="p">)</span>
+
+ <span class="n">url</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span> <span class="o">+</span> <span class="n">endpoint</span>
+ <span class="n">req</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s1">'GET'</span><span class="p">:</span>
+ <span class="c1"># GET uses params</span>
+ <span class="n">req</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">Request</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">method</span><span class="p">,</span>
+ <span class="n">url</span><span class="p">,</span>
+ <span class="n">params</span><span class="o">=</span><span class="n">data</span><span class="p">,</span>
+ <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s1">'HEAD'</span><span class="p">:</span>
+ <span class="c1"># HEAD doesn't use params</span>
+ <span class="n">req</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">Request</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">method</span><span class="p">,</span>
+ <span class="n">url</span><span class="p">,</span>
+ <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="c1"># Others use data</span>
+ <span class="n">req</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">Request</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">method</span><span class="p">,</span>
+ <span class="n">url</span><span class="p">,</span>
+ <span class="n">data</span><span class="o">=</span><span class="n">data</span><span class="p">,</span>
+ <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span>
+
+ <span class="n">prepped_request</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">prepare_request</span><span class="p">(</span><span class="n">req</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Sending '</span><span class="si">%s</span><span class="s2">' to url: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_and_check</span><span class="p">(</span><span class="n">session</span><span class="p">,</span> <span class="n">prepped_request</span><span class="p">,</span> <span class="n">extra_options</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="HttpHook.run_and_check"><a class="viewcode-back" href="../../../code.html#airflow.hooks.http_hook.HttpHook.run_and_check">[docs]</a> <span class="k">def</span> <span class="nf">run_and_check</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="p">,</span> <span class="n">prepped_request</span><span class="p">,</span> <span class="n">extra_options</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Grabs extra options like timeout and actually runs the request,</span>
+<span class="sd"> checking for the result</span>
+<span class="sd"> """</span>
+ <span class="n">extra_options</span> <span class="o">=</span> <span class="n">extra_options</span> <span class="ow">or</span> <span class="p">{}</span>
+
+ <span class="n">response</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">send</span><span class="p">(</span>
+ <span class="n">prepped_request</span><span class="p">,</span>
+ <span class="n">stream</span><span class="o">=</span><span class="n">extra_options</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"stream"</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
+ <span class="n">verify</span><span class="o">=</span><span class="n">extra_options</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"verify"</span><span class="p">,</span> <span class="kc">False</span><span class="p">),</span>
+ <span class="n">proxies</span><span class="o">=</span><span class="n">extra_options</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"proxies"</span><span class="p">,</span> <span class="p">{}),</span>
+ <span class="n">cert</span><span class="o">=</span><span class="n">extra_options</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"cert"</span><span class="p">),</span>
+ <span class="n">timeout</span><span class="o">=</span><span class="n">extra_options</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"timeout"</span><span class="p">),</span>
+ <span class="n">allow_redirects</span><span class="o">=</span><span class="n">extra_options</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"allow_redirects"</span><span class="p">,</span> <span class="kc">True</span><span class="p">))</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">response</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
+ <span class="k">except</span> <span class="n">requests</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">HTTPError</span><span class="p">:</span>
+ <span class="c1"># Tried rewrapping, but not supported. This way, it's possible</span>
+ <span class="c1"># to get reason and code for failure by checking first 3 chars</span>
+ <span class="c1"># for the code, or do a split on ':'</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"HTTP error: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">response</span><span class="o">.</span><span class="n">reason</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'GET'</span><span class="p">,</span> <span class="s1">'HEAD'</span><span class="p">):</span>
+ <span class="c1"># The sensor uses GET, so this prevents filling up the log</span>
+ <span class="c1"># with the body every time the GET 'misses'.</span>
+ <span class="c1"># That's ok to do, because GETs should be repeatable and</span>
+ <span class="c1"># all data should be visible in the log (no post data)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">status_code</span><span class="p">)</span><span class="o">+</span><span class="s2">":"</span><span class="o">+</span><span class="n">response</span><span class="o">.</span><span class="n">reason</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">response</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/hooks/mssql_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/hooks/mssql_hook.html b/_modules/airflow/hooks/mssql_hook.html
new file mode 100644
index 0000000..dfb4ce8
--- /dev/null
+++ b/_modules/airflow/hooks/mssql_hook.html
@@ -0,0 +1,287 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.hooks.mssql_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.hooks.mssql_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.hooks.mssql_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">pymssql</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="k">import</span> <span class="n">DbApiHook</span>
+
+
+<div class="viewcode-block" id="MsSqlHook"><a class="viewcode-back" href="../../../code.html#airflow.hooks.mssql_hook.MsSqlHook">[docs]</a><span class="k">class</span> <span class="nc">MsSqlHook</span><span class="p">(</span><span class="n">DbApiHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with Microsoft SQL Server.</span>
+<span class="sd"> """</span>
+
+ <span class="n">conn_name_attr</span> <span class="o">=</span> <span class="s1">'mssql_conn_id'</span>
+ <span class="n">default_conn_name</span> <span class="o">=</span> <span class="s1">'mssql_default'</span>
+ <span class="n">supports_autocommit</span> <span class="o">=</span> <span class="kc">True</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">MsSqlHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s2">"schema"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+
+<div class="viewcode-block" id="MsSqlHook.get_conn"><a class="viewcode-back" href="../../../code.html#airflow.hooks.mssql_hook.MsSqlHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns a mssql connection object</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mssql_conn_id</span><span class="p">)</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="n">pymssql</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span>
+ <span class="n">server</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">,</span>
+ <span class="n">user</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span>
+ <span class="n">password</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span>
+ <span class="n">database</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="ow">or</span> <span class="n">conn</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span>
+ <span class="n">port</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">conn</span></div>
+
+ <span class="k">def</span> <span class="nf">set_autocommit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn</span><span class="p">,</span> <span class="n">autocommit</span><span class="p">):</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">autocommit</span><span class="p">(</span><span class="n">autocommit</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/hooks/mysql_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/hooks/mysql_hook.html b/_modules/airflow/hooks/mysql_hook.html
new file mode 100644
index 0000000..ae309f5
--- /dev/null
+++ b/_modules/airflow/hooks/mysql_hook.html
@@ -0,0 +1,341 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.hooks.mysql_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.hooks.mysql_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.hooks.mysql_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">MySQLdb</span>
+<span class="kn">import</span> <span class="nn">MySQLdb.cursors</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="k">import</span> <span class="n">DbApiHook</span>
+
+
+<div class="viewcode-block" id="MySqlHook"><a class="viewcode-back" href="../../../code.html#airflow.hooks.mysql_hook.MySqlHook">[docs]</a><span class="k">class</span> <span class="nc">MySqlHook</span><span class="p">(</span><span class="n">DbApiHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with MySQL.</span>
+
+<span class="sd"> You can specify charset in the extra field of your connection</span>
+<span class="sd"> as ``{"charset": "utf8"}``. Also you can choose cursor as</span>
+<span class="sd"> ``{"cursor": "SSCursor"}``. Refer to the MySQLdb.cursors for more details.</span>
+<span class="sd"> """</span>
+
+ <span class="n">conn_name_attr</span> <span class="o">=</span> <span class="s1">'mysql_conn_id'</span>
+ <span class="n">default_conn_name</span> <span class="o">=</span> <span class="s1">'mysql_default'</span>
+ <span class="n">supports_autocommit</span> <span class="o">=</span> <span class="kc">True</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">MySqlHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s2">"schema"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+
+<div class="viewcode-block" id="MySqlHook.get_conn"><a class="viewcode-back" href="../../../code.html#airflow.hooks.mysql_hook.MySqlHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns a mysql connection object</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mysql_conn_id</span><span class="p">)</span>
+ <span class="n">conn_config</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s2">"user"</span><span class="p">:</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span>
+ <span class="s2">"passwd"</span><span class="p">:</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span> <span class="ow">or</span> <span class="s1">''</span><span class="p">,</span>
+ <span class="s2">"host"</span><span class="p">:</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span> <span class="ow">or</span> <span class="s1">'localhost'</span><span class="p">,</span>
+ <span class="s2">"db"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="ow">or</span> <span class="n">conn</span><span class="o">.</span><span class="n">schema</span> <span class="ow">or</span> <span class="s1">''</span>
+ <span class="p">}</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">:</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s2">"port"</span><span class="p">]</span> <span class="o">=</span> <span class="mi">3306</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s2">"port"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'charset'</span><span class="p">,</span> <span class="kc">False</span><span class="p">):</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s2">"charset"</span><span class="p">]</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s2">"charset"</span><span class="p">]</span>
+ <span class="k">if</span> <span class="p">(</span><span class="n">conn_config</span><span class="p">[</span><span class="s2">"charset"</span><span class="p">])</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'utf8'</span> <span class="ow">or</span>\
+ <span class="p">(</span><span class="n">conn_config</span><span class="p">[</span><span class="s2">"charset"</span><span class="p">])</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'utf-8'</span><span class="p">:</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s2">"use_unicode"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'cursor'</span><span class="p">,</span> <span class="kc">False</span><span class="p">):</span>
+ <span class="k">if</span> <span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s2">"cursor"</span><span class="p">])</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'sscursor'</span><span class="p">:</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s2">"cursorclass"</span><span class="p">]</span> <span class="o">=</span> <span class="n">MySQLdb</span><span class="o">.</span><span class="n">cursors</span><span class="o">.</span><span class="n">SSCursor</span>
+ <span class="k">elif</span> <span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s2">"cursor"</span><span class="p">])</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'dictcursor'</span><span class="p">:</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s2">"cursorclass"</span><span class="p">]</span> <span class="o">=</span> <span class="n">MySQLdb</span><span class="o">.</span><span class="n">cursors</span><span class="o">.</span><span class="n">DictCursor</span>
+ <span class="k">elif</span> <span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s2">"cursor"</span><span class="p">])</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'ssdictcursor'</span><span class="p">:</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s2">"cursorclass"</span><span class="p">]</span> <span class="o">=</span> <span class="n">MySQLdb</span><span class="o">.</span><span class="n">cursors</span><span class="o">.</span><span class="n">SSDictCursor</span>
+ <span class="n">local_infile</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'local_infile'</span><span class="p">,</span><span class="kc">False</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'ssl'</span><span class="p">,</span> <span class="kc">False</span><span class="p">):</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s1">'ssl'</span><span class="p">]</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s1">'ssl'</span><span class="p">]</span>
+ <span class="k">if</span> <span class="n">local_infile</span><span class="p">:</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s2">"local_infile"</span><span class="p">]</span> <span class="o">=</span> <span class="mi">1</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="n">MySQLdb</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="o">**</span><span class="n">conn_config</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">conn</span></div>
+
+<div class="viewcode-block" id="MySqlHook.bulk_load"><a class="viewcode-back" href="../../../code.html#airflow.hooks.mysql_hook.MySqlHook.bulk_load">[docs]</a> <span class="k">def</span> <span class="nf">bulk_load</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">tmp_file</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Loads a tab-delimited file into a database table</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">cur</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
+ <span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="s2">"""</span>
+<span class="s2"> LOAD DATA LOCAL INFILE '</span><span class="si">{tmp_file}</span><span class="s2">'</span>
+<span class="s2"> INTO TABLE </span><span class="si">{table}</span><span class="s2"></span>
+<span class="s2"> """</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span></div>
+
+ <span class="nd">@staticmethod</span>
+ <span class="k">def</span> <span class="nf">_serialize_cell</span><span class="p">(</span><span class="n">cell</span><span class="p">,</span> <span class="n">conn</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> MySQLdb converts an argument to a literal when passing those seperately to execute.</span>
+<span class="sd"> Hence, this method does nothing.</span>
+
+<span class="sd"> :param cell: The cell to insert into the table</span>
+<span class="sd"> :type cell: object</span>
+<span class="sd"> :param conn: The database connection</span>
+<span class="sd"> :type conn: connection object</span>
+<span class="sd"> :return: The same cell</span>
+<span class="sd"> :rtype: object</span>
+<span class="sd"> """</span>
+
+ <span class="k">return</span> <span class="n">cell</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[48/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/databricks_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/databricks_hook.html b/_modules/airflow/contrib/hooks/databricks_hook.html
new file mode 100644
index 0000000..00ae02f
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/databricks_hook.html
@@ -0,0 +1,462 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.databricks_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.databricks_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.databricks_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+<span class="kn">import</span> <span class="nn">requests</span>
+
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">__version__</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">requests</span> <span class="k">import</span> <span class="n">exceptions</span> <span class="k">as</span> <span class="n">requests_exceptions</span>
+<span class="kn">from</span> <span class="nn">requests.auth</span> <span class="k">import</span> <span class="n">AuthBase</span>
+
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+<span class="k">try</span><span class="p">:</span>
+ <span class="kn">from</span> <span class="nn">urllib</span> <span class="k">import</span> <span class="n">parse</span> <span class="k">as</span> <span class="n">urlparse</span>
+<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
+ <span class="kn">import</span> <span class="nn">urlparse</span>
+
+
+<span class="n">SUBMIT_RUN_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'POST'</span><span class="p">,</span> <span class="s1">'api/2.0/jobs/runs/submit'</span><span class="p">)</span>
+<span class="n">GET_RUN_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'GET'</span><span class="p">,</span> <span class="s1">'api/2.0/jobs/runs/get'</span><span class="p">)</span>
+<span class="n">CANCEL_RUN_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'POST'</span><span class="p">,</span> <span class="s1">'api/2.0/jobs/runs/cancel'</span><span class="p">)</span>
+<span class="n">USER_AGENT_HEADER</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'user-agent'</span><span class="p">:</span> <span class="s1">'airflow-</span><span class="si">{v}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">v</span><span class="o">=</span><span class="n">__version__</span><span class="p">)}</span>
+
+
+<div class="viewcode-block" id="DatabricksHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.databricks_hook.DatabricksHook">[docs]</a><span class="k">class</span> <span class="nc">DatabricksHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with Databricks.</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">databricks_conn_id</span><span class="o">=</span><span class="s1">'databricks_default'</span><span class="p">,</span>
+ <span class="n">timeout_seconds</span><span class="o">=</span><span class="mi">180</span><span class="p">,</span>
+ <span class="n">retry_limit</span><span class="o">=</span><span class="mi">3</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> :param databricks_conn_id: The name of the databricks connection to use.</span>
+<span class="sd"> :type databricks_conn_id: string</span>
+<span class="sd"> :param timeout_seconds: The amount of time in seconds the requests library</span>
+<span class="sd"> will wait before timing-out.</span>
+<span class="sd"> :type timeout_seconds: int</span>
+<span class="sd"> :param retry_limit: The number of times to retry the connection in case of</span>
+<span class="sd"> service outages.</span>
+<span class="sd"> :type retry_limit: int</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn_id</span> <span class="o">=</span> <span class="n">databricks_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">databricks_conn_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">timeout_seconds</span> <span class="o">=</span> <span class="n">timeout_seconds</span>
+ <span class="k">assert</span> <span class="n">retry_limit</span> <span class="o">>=</span> <span class="mi">1</span><span class="p">,</span> <span class="s1">'Retry limit must be greater than equal to 1'</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">retry_limit</span> <span class="o">=</span> <span class="n">retry_limit</span>
+
+ <span class="k">def</span> <span class="nf">_parse_host</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">host</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> The purpose of this function is to be robust to improper connections</span>
+<span class="sd"> settings provided by users, specifically in the host field.</span>
+
+
+<span class="sd"> For example -- when users supply ``https://xx.cloud.databricks.com`` as the</span>
+<span class="sd"> host, we must strip out the protocol to get the host.</span>
+<span class="sd"> >>> h = DatabricksHook()</span>
+<span class="sd"> >>> assert h._parse_host('https://xx.cloud.databricks.com') == \</span>
+<span class="sd"> 'xx.cloud.databricks.com'</span>
+
+<span class="sd"> In the case where users supply the correct ``xx.cloud.databricks.com`` as the</span>
+<span class="sd"> host, this function is a no-op.</span>
+<span class="sd"> >>> assert h._parse_host('xx.cloud.databricks.com') == 'xx.cloud.databricks.com'</span>
+<span class="sd"> """</span>
+ <span class="n">urlparse_host</span> <span class="o">=</span> <span class="n">urlparse</span><span class="o">.</span><span class="n">urlparse</span><span class="p">(</span><span class="n">host</span><span class="p">)</span><span class="o">.</span><span class="n">hostname</span>
+ <span class="k">if</span> <span class="n">urlparse_host</span><span class="p">:</span>
+ <span class="c1"># In this case, host = https://xx.cloud.databricks.com</span>
+ <span class="k">return</span> <span class="n">urlparse_host</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="c1"># In this case, host = xx.cloud.databricks.com</span>
+ <span class="k">return</span> <span class="n">host</span>
+
+ <span class="k">def</span> <span class="nf">_do_api_call</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">endpoint_info</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Utility function to perform an API call with retries</span>
+<span class="sd"> :param endpoint_info: Tuple of method and endpoint</span>
+<span class="sd"> :type endpoint_info: (string, string)</span>
+<span class="sd"> :param json: Parameters for this API call.</span>
+<span class="sd"> :type json: dict</span>
+<span class="sd"> :return: If the api call returns a OK status code,</span>
+<span class="sd"> this function returns the response in JSON. Otherwise,</span>
+<span class="sd"> we throw an AirflowException.</span>
+<span class="sd"> :rtype: dict</span>
+<span class="sd"> """</span>
+ <span class="n">method</span><span class="p">,</span> <span class="n">endpoint</span> <span class="o">=</span> <span class="n">endpoint_info</span>
+ <span class="n">url</span> <span class="o">=</span> <span class="s1">'https://</span><span class="si">{host}</span><span class="s1">/</span><span class="si">{endpoint}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">host</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_parse_host</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn</span><span class="o">.</span><span class="n">host</span><span class="p">),</span>
+ <span class="n">endpoint</span><span class="o">=</span><span class="n">endpoint</span><span class="p">)</span>
+ <span class="k">if</span> <span class="s1">'token'</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Using token auth.'</span><span class="p">)</span>
+ <span class="n">auth</span> <span class="o">=</span> <span class="n">_TokenAuth</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s1">'token'</span><span class="p">])</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Using basic auth.'</span><span class="p">)</span>
+ <span class="n">auth</span> <span class="o">=</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn</span><span class="o">.</span><span class="n">password</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">method</span> <span class="o">==</span> <span class="s1">'GET'</span><span class="p">:</span>
+ <span class="n">request_func</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">get</span>
+ <span class="k">elif</span> <span class="n">method</span> <span class="o">==</span> <span class="s1">'POST'</span><span class="p">:</span>
+ <span class="n">request_func</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">post</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Unexpected HTTP Method: '</span> <span class="o">+</span> <span class="n">method</span><span class="p">)</span>
+
+ <span class="k">for</span> <span class="n">attempt_num</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">retry_limit</span><span class="o">+</span><span class="mi">1</span><span class="p">):</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="n">request_func</span><span class="p">(</span>
+ <span class="n">url</span><span class="p">,</span>
+ <span class="n">json</span><span class="o">=</span><span class="n">json</span><span class="p">,</span>
+ <span class="n">auth</span><span class="o">=</span><span class="n">auth</span><span class="p">,</span>
+ <span class="n">headers</span><span class="o">=</span><span class="n">USER_AGENT_HEADER</span><span class="p">,</span>
+ <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout_seconds</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="n">requests</span><span class="o">.</span><span class="n">codes</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="c1"># In this case, the user probably made a mistake.</span>
+ <span class="c1"># Don't retry.</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Response: </span><span class="si">{0}</span><span class="s1">, Status Code: </span><span class="si">{1}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">response</span><span class="o">.</span><span class="n">content</span><span class="p">,</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span><span class="p">))</span>
+ <span class="k">except</span> <span class="p">(</span><span class="n">requests_exceptions</span><span class="o">.</span><span class="n">ConnectionError</span><span class="p">,</span>
+ <span class="n">requests_exceptions</span><span class="o">.</span><span class="n">Timeout</span><span class="p">)</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
+ <span class="s1">'Attempt </span><span class="si">%s</span><span class="s1"> API Request to Databricks failed with reason: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">attempt_num</span><span class="p">,</span> <span class="n">e</span>
+ <span class="p">)</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">((</span><span class="s1">'API requests to Databricks failed </span><span class="si">{}</span><span class="s1"> times. '</span> <span class="o">+</span>
+ <span class="s1">'Giving up.'</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">retry_limit</span><span class="p">))</span>
+
+<div class="viewcode-block" id="DatabricksHook.submit_run"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.databricks_hook.DatabricksHook.submit_run">[docs]</a> <span class="k">def</span> <span class="nf">submit_run</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Utility function to call the ``api/2.0/jobs/runs/submit`` endpoint.</span>
+
+<span class="sd"> :param json: The data used in the body of the request to the ``submit`` endpoint.</span>
+<span class="sd"> :type json: dict</span>
+<span class="sd"> :return: the run_id as a string</span>
+<span class="sd"> :rtype: string</span>
+<span class="sd"> """</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">SUBMIT_RUN_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">response</span><span class="p">[</span><span class="s1">'run_id'</span><span class="p">]</span></div>
+
+ <span class="k">def</span> <span class="nf">get_run_page_url</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">run_id</span><span class="p">):</span>
+ <span class="n">json</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'run_id'</span><span class="p">:</span> <span class="n">run_id</span><span class="p">}</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">GET_RUN_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">response</span><span class="p">[</span><span class="s1">'run_page_url'</span><span class="p">]</span>
+
+ <span class="k">def</span> <span class="nf">get_run_state</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">run_id</span><span class="p">):</span>
+ <span class="n">json</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'run_id'</span><span class="p">:</span> <span class="n">run_id</span><span class="p">}</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">GET_RUN_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span>
+ <span class="n">state</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">'state'</span><span class="p">]</span>
+ <span class="n">life_cycle_state</span> <span class="o">=</span> <span class="n">state</span><span class="p">[</span><span class="s1">'life_cycle_state'</span><span class="p">]</span>
+ <span class="c1"># result_state may not be in the state if not terminal</span>
+ <span class="n">result_state</span> <span class="o">=</span> <span class="n">state</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'result_state'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="n">state_message</span> <span class="o">=</span> <span class="n">state</span><span class="p">[</span><span class="s1">'state_message'</span><span class="p">]</span>
+ <span class="k">return</span> <span class="n">RunState</span><span class="p">(</span><span class="n">life_cycle_state</span><span class="p">,</span> <span class="n">result_state</span><span class="p">,</span> <span class="n">state_message</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">cancel_run</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">run_id</span><span class="p">):</span>
+ <span class="n">json</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'run_id'</span><span class="p">:</span> <span class="n">run_id</span><span class="p">}</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">CANCEL_RUN_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span></div>
+
+
+<span class="n">RUN_LIFE_CYCLE_STATES</span> <span class="o">=</span> <span class="p">[</span>
+ <span class="s1">'PENDING'</span><span class="p">,</span>
+ <span class="s1">'RUNNING'</span><span class="p">,</span>
+ <span class="s1">'TERMINATING'</span><span class="p">,</span>
+ <span class="s1">'TERMINATED'</span><span class="p">,</span>
+ <span class="s1">'SKIPPED'</span><span class="p">,</span>
+ <span class="s1">'INTERNAL_ERROR'</span>
+<span class="p">]</span>
+
+
+<span class="k">class</span> <span class="nc">RunState</span><span class="p">:</span>
+ <span class="sd">"""</span>
+<span class="sd"> Utility class for the run state concept of Databricks runs.</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">life_cycle_state</span><span class="p">,</span> <span class="n">result_state</span><span class="p">,</span> <span class="n">state_message</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">life_cycle_state</span> <span class="o">=</span> <span class="n">life_cycle_state</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">result_state</span> <span class="o">=</span> <span class="n">result_state</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">state_message</span> <span class="o">=</span> <span class="n">state_message</span>
+
+ <span class="nd">@property</span>
+ <span class="k">def</span> <span class="nf">is_terminal</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">life_cycle_state</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">RUN_LIFE_CYCLE_STATES</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">((</span><span class="s1">'Unexpected life cycle state: </span><span class="si">{}</span><span class="s1">: If the state has '</span>
+ <span class="s1">'been introduced recently, please check the Databricks user '</span>
+ <span class="s1">'guide for troubleshooting information'</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">life_cycle_state</span><span class="p">))</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">life_cycle_state</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'TERMINATED'</span><span class="p">,</span> <span class="s1">'SKIPPED'</span><span class="p">,</span> <span class="s1">'INTERNAL_ERROR'</span><span class="p">)</span>
+
+ <span class="nd">@property</span>
+ <span class="k">def</span> <span class="nf">is_successful</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_state</span> <span class="o">==</span> <span class="s1">'SUCCESS'</span>
+
+ <span class="k">def</span> <span class="nf">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">life_cycle_state</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">life_cycle_state</span> <span class="ow">and</span> \
+ <span class="bp">self</span><span class="o">.</span><span class="n">result_state</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">result_state</span> <span class="ow">and</span> \
+ <span class="bp">self</span><span class="o">.</span><span class="n">state_message</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">state_message</span>
+
+ <span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">_TokenAuth</span><span class="p">(</span><span class="n">AuthBase</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Helper class for requests Auth field. AuthBase requires you to implement the __call__</span>
+<span class="sd"> magic function.</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">token</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">token</span> <span class="o">=</span> <span class="n">token</span>
+
+ <span class="k">def</span> <span class="nf">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">r</span><span class="p">):</span>
+ <span class="n">r</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="s1">'Authorization'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'Bearer '</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">token</span>
+ <span class="k">return</span> <span class="n">r</span>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/datadog_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/datadog_hook.html b/_modules/airflow/contrib/hooks/datadog_hook.html
new file mode 100644
index 0000000..b8c8e04
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/datadog_hook.html
@@ -0,0 +1,375 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.datadog_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.datadog_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.datadog_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">time</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">datadog</span> <span class="k">import</span> <span class="n">initialize</span><span class="p">,</span> <span class="n">api</span>
+
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+
+<div class="viewcode-block" id="DatadogHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.datadog_hook.DatadogHook">[docs]</a><span class="k">class</span> <span class="nc">DatadogHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Uses datadog API to send metrics of practically anything measurable,</span>
+<span class="sd"> so it's possible to track # of db records inserted/deleted, records read</span>
+<span class="sd"> from file and many other useful metrics.</span>
+
+<span class="sd"> Depends on the datadog API, which has to be deployed on the same server where</span>
+<span class="sd"> Airflow runs.</span>
+
+<span class="sd"> :param datadog_conn_id: The connection to datadog, containing metadata for api keys.</span>
+<span class="sd"> :param datadog_conn_id: string</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">datadog_conn_id</span><span class="o">=</span><span class="s1">'datadog_default'</span><span class="p">):</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">datadog_conn_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">api_key</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'api_key'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">app_key</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'app_key'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">source_type_name</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'source_type_name'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+
+ <span class="c1"># If the host is populated, it will use that hostname instead.</span>
+ <span class="c1"># for all metric submissions.</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">host</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">api_key</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"api_key must be specified in the Datadog connection details"</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">app_key</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"app_key must be specified in the Datadog connection details"</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Setting up api keys for Datadog"</span><span class="p">)</span>
+ <span class="n">options</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'api_key'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">api_key</span><span class="p">,</span>
+ <span class="s1">'app_key'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">app_key</span>
+ <span class="p">}</span>
+ <span class="n">initialize</span><span class="p">(</span><span class="o">**</span><span class="n">options</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">validate_response</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">response</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s1">'status'</span><span class="p">]</span> <span class="o">!=</span> <span class="s1">'ok'</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Datadog returned: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">response</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Error status received from Datadog"</span><span class="p">)</span>
+
+<div class="viewcode-block" id="DatadogHook.send_metric"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.datadog_hook.DatadogHook.send_metric">[docs]</a> <span class="k">def</span> <span class="nf">send_metric</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">metric_name</span><span class="p">,</span> <span class="n">datapoint</span><span class="p">,</span> <span class="n">tags</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Sends a single datapoint metric to DataDog</span>
+
+<span class="sd"> :param metric_name: The name of the metric</span>
+<span class="sd"> :type metric_name: string</span>
+<span class="sd"> :param datapoint: A single integer or float related to the metric</span>
+<span class="sd"> :type datapoint: integer or float</span>
+<span class="sd"> :param tags: A list of tags associated with the metric</span>
+<span class="sd"> :type tags: list</span>
+<span class="sd"> """</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="n">api</span><span class="o">.</span><span class="n">Metric</span><span class="o">.</span><span class="n">send</span><span class="p">(</span>
+ <span class="n">metric</span><span class="o">=</span><span class="n">metric_name</span><span class="p">,</span>
+ <span class="n">points</span><span class="o">=</span><span class="n">datapoint</span><span class="p">,</span>
+ <span class="n">host</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">host</span><span class="p">,</span>
+ <span class="n">tags</span><span class="o">=</span><span class="n">tags</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">validate_response</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">response</span></div>
+
+<div class="viewcode-block" id="DatadogHook.query_metric"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.datadog_hook.DatadogHook.query_metric">[docs]</a> <span class="k">def</span> <span class="nf">query_metric</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">query</span><span class="p">,</span>
+ <span class="n">from_seconds_ago</span><span class="p">,</span>
+ <span class="n">to_seconds_ago</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Queries datadog for a specific metric, potentially with some function applied to it</span>
+<span class="sd"> and returns the results.</span>
+
+<span class="sd"> :param query: The datadog query to execute (see datadog docs)</span>
+<span class="sd"> :type query: string</span>
+<span class="sd"> :param from_seconds_ago: How many seconds ago to start querying for.</span>
+<span class="sd"> :type from_seconds_ago: int</span>
+<span class="sd"> :param to_seconds_ago: Up to how many seconds ago to query for.</span>
+<span class="sd"> :type to_seconds_ago: int</span>
+<span class="sd"> """</span>
+ <span class="n">now</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">())</span>
+
+ <span class="n">response</span> <span class="o">=</span> <span class="n">api</span><span class="o">.</span><span class="n">Metric</span><span class="o">.</span><span class="n">query</span><span class="p">(</span>
+ <span class="n">start</span><span class="o">=</span><span class="n">now</span> <span class="o">-</span> <span class="n">from_seconds_ago</span><span class="p">,</span>
+ <span class="n">end</span><span class="o">=</span><span class="n">now</span> <span class="o">-</span> <span class="n">to_seconds_ago</span><span class="p">,</span>
+ <span class="n">query</span><span class="o">=</span><span class="n">query</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">validate_response</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">response</span></div>
+
+<div class="viewcode-block" id="DatadogHook.post_event"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.datadog_hook.DatadogHook.post_event">[docs]</a> <span class="k">def</span> <span class="nf">post_event</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">title</span><span class="p">,</span> <span class="n">text</span><span class="p">,</span> <span class="n">tags</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">alert_type</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">aggregation_key</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Posts an event to datadog (processing finished, potentially alerts, other issues)</span>
+<span class="sd"> Think about this as a means to maintain persistence of alerts, rather than alerting</span>
+<span class="sd"> itself.</span>
+
+<span class="sd"> :param title: The title of the event</span>
+<span class="sd"> :type title: string</span>
+<span class="sd"> :param text: The body of the event (more information)</span>
+<span class="sd"> :type text: string</span>
+<span class="sd"> :param tags: List of string tags to apply to the event</span>
+<span class="sd"> :type tags: list</span>
+<span class="sd"> :param alert_type: The alert type for the event, one of</span>
+<span class="sd"> ["error", "warning", "info", "success"]</span>
+<span class="sd"> :type alert_type: string</span>
+<span class="sd"> :param aggregation_key: Key that can be used to aggregate this event in a stream</span>
+<span class="sd"> :type aggregation_key: string</span>
+<span class="sd"> """</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="n">api</span><span class="o">.</span><span class="n">Event</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
+ <span class="n">title</span><span class="o">=</span><span class="n">title</span><span class="p">,</span>
+ <span class="n">text</span><span class="o">=</span><span class="n">text</span><span class="p">,</span>
+ <span class="n">host</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">host</span><span class="p">,</span>
+ <span class="n">tags</span><span class="o">=</span><span class="n">tags</span><span class="p">,</span>
+ <span class="n">alert_type</span><span class="o">=</span><span class="n">alert_type</span><span class="p">,</span>
+ <span class="n">aggregation_key</span><span class="o">=</span><span class="n">aggregation_key</span><span class="p">,</span>
+ <span class="n">source_type_name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">source_type_name</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">validate_response</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">response</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/datastore_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/datastore_hook.html b/_modules/airflow/contrib/hooks/datastore_hook.html
index 21363c3..24ae17e 100644
--- a/_modules/airflow/contrib/hooks/datastore_hook.html
+++ b/_modules/airflow/contrib/hooks/datastore_hook.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,22 @@
<h1>Source code for airflow.contrib.hooks.datastore_hook</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="c1">#</span>
<span class="kn">import</span> <span class="nn">json</span>
@@ -194,7 +201,7 @@
<span class="sd"> connection.</span>
<span class="sd"> This object is not threads safe. If you want to make multiple requests</span>
-<span class="sd"> simultaniously, you will need to create a hook per thread.</span>
+<span class="sd"> simultaneously, you will need to create a hook per thread.</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
@@ -235,7 +242,9 @@
<div class="viewcode-block" id="DatastoreHook.commit"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.commit">[docs]</a> <span class="k">def</span> <span class="nf">commit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">body</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Commit a transaction, optionally creating, deleting or modifying some entities.</span>
-<span class="sd"> see https://cloud.google.com/datastore/docs/reference/rest/v1/projects/commit</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> https://cloud.google.com/datastore/docs/reference/rest/v1/projects/commit</span>
<span class="sd"> :param body: the body of the commit request</span>
<span class="sd"> :return: the response body of the commit request</span>
@@ -246,7 +255,10 @@
<div class="viewcode-block" id="DatastoreHook.lookup"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.lookup">[docs]</a> <span class="k">def</span> <span class="nf">lookup</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">keys</span><span class="p">,</span> <span class="n">read_consistency</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">transaction</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Lookup some entities by key</span>
-<span class="sd"> see https://cloud.google.com/datastore/docs/reference/rest/v1/projects/lookup</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> https://cloud.google.com/datastore/docs/reference/rest/v1/projects/lookup</span>
+
<span class="sd"> :param keys: the keys to lookup</span>
<span class="sd"> :param read_consistency: the read consistency to use. default, strong or eventual.</span>
<span class="sd"> Cannot be used with a transaction.</span>
@@ -263,7 +275,10 @@
<div class="viewcode-block" id="DatastoreHook.rollback"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.rollback">[docs]</a> <span class="k">def</span> <span class="nf">rollback</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">transaction</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Roll back a transaction</span>
-<span class="sd"> see https://cloud.google.com/datastore/docs/reference/rest/v1/projects/rollback</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> https://cloud.google.com/datastore/docs/reference/rest/v1/projects/rollback</span>
+
<span class="sd"> :param transaction: the transaction to roll back</span>
<span class="sd"> """</span>
<span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">rollback</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="p">{</span><span class="s1">'transaction'</span><span class="p">:</span> <span class="n">transaction</span><span class="p">})</span>\
@@ -272,7 +287,10 @@
<div class="viewcode-block" id="DatastoreHook.run_query"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.run_query">[docs]</a> <span class="k">def</span> <span class="nf">run_query</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">body</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Run a query for entities.</span>
-<span class="sd"> see https://cloud.google.com/datastore/docs/reference/rest/v1/projects/runQuery</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> https://cloud.google.com/datastore/docs/reference/rest/v1/projects/runQuery</span>
+
<span class="sd"> :param body: the body of the query request</span>
<span class="sd"> :return: the batch of query results.</span>
<span class="sd"> """</span>
[41/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/redshift_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/redshift_hook.html b/_modules/airflow/contrib/hooks/redshift_hook.html
new file mode 100644
index 0000000..314fd9e
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/redshift_hook.html
@@ -0,0 +1,348 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.redshift_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.redshift_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.redshift_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.aws_hook</span> <span class="k">import</span> <span class="n">AwsHook</span>
+
+
+<div class="viewcode-block" id="RedshiftHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.redshift_hook.RedshiftHook">[docs]</a><span class="k">class</span> <span class="nc">RedshiftHook</span><span class="p">(</span><span class="n">AwsHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with AWS Redshift, using the boto3 library</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_client_type</span><span class="p">(</span><span class="s1">'redshift'</span><span class="p">)</span>
+
+ <span class="c1"># TODO: Wrap create_cluster_snapshot</span>
+<div class="viewcode-block" id="RedshiftHook.cluster_status"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.redshift_hook.RedshiftHook.cluster_status">[docs]</a> <span class="k">def</span> <span class="nf">cluster_status</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cluster_identifier</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Return status of a cluster</span>
+
+<span class="sd"> :param cluster_identifier: unique identifier of a cluster</span>
+<span class="sd"> :type cluster_identifier: str</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">describe_clusters</span><span class="p">(</span>
+ <span class="n">ClusterIdentifier</span><span class="o">=</span><span class="n">cluster_identifier</span><span class="p">)[</span><span class="s1">'Clusters'</span><span class="p">]</span>
+ <span class="k">return</span> <span class="n">response</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s1">'ClusterStatus'</span><span class="p">]</span> <span class="k">if</span> <span class="n">response</span> <span class="k">else</span> <span class="kc">None</span>
+ <span class="k">except</span> <span class="n">conn</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">ClusterNotFoundFault</span><span class="p">:</span>
+ <span class="k">return</span> <span class="s1">'cluster_not_found'</span></div>
+
+<div class="viewcode-block" id="RedshiftHook.delete_cluster"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.redshift_hook.RedshiftHook.delete_cluster">[docs]</a> <span class="k">def</span> <span class="nf">delete_cluster</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">cluster_identifier</span><span class="p">,</span>
+ <span class="n">skip_final_cluster_snapshot</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+ <span class="n">final_cluster_snapshot_identifier</span><span class="o">=</span><span class="s1">''</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Delete a cluster and optionally create a snapshot</span>
+
+<span class="sd"> :param cluster_identifier: unique identifier of a cluster</span>
+<span class="sd"> :type cluster_identifier: str</span>
+<span class="sd"> :param skip_final_cluster_snapshot: determines cluster snapshot creation</span>
+<span class="sd"> :type skip_final_cluster_snapshot: bool</span>
+<span class="sd"> :param final_cluster_snapshot_identifier: name of final cluster snapshot</span>
+<span class="sd"> :type final_cluster_snapshot_identifier: str</span>
+<span class="sd"> """</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">delete_cluster</span><span class="p">(</span>
+ <span class="n">ClusterIdentifier</span><span class="o">=</span><span class="n">cluster_identifier</span><span class="p">,</span>
+ <span class="n">SkipFinalClusterSnapshot</span><span class="o">=</span><span class="n">skip_final_cluster_snapshot</span><span class="p">,</span>
+ <span class="n">FinalClusterSnapshotIdentifier</span><span class="o">=</span><span class="n">final_cluster_snapshot_identifier</span>
+ <span class="p">)</span>
+ <span class="k">return</span> <span class="n">response</span><span class="p">[</span><span class="s1">'Cluster'</span><span class="p">]</span> <span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s1">'Cluster'</span><span class="p">]</span> <span class="k">else</span> <span class="kc">None</span></div>
+
+<div class="viewcode-block" id="RedshiftHook.describe_cluster_snapshots"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.redshift_hook.RedshiftHook.describe_cluster_snapshots">[docs]</a> <span class="k">def</span> <span class="nf">describe_cluster_snapshots</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cluster_identifier</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Gets a list of snapshots for a cluster</span>
+
+<span class="sd"> :param cluster_identifier: unique identifier of a cluster</span>
+<span class="sd"> :type cluster_identifier: str</span>
+<span class="sd"> """</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">describe_cluster_snapshots</span><span class="p">(</span>
+ <span class="n">ClusterIdentifier</span><span class="o">=</span><span class="n">cluster_identifier</span>
+ <span class="p">)</span>
+ <span class="k">if</span> <span class="s1">'Snapshots'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">response</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">None</span>
+ <span class="n">snapshots</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">'Snapshots'</span><span class="p">]</span>
+ <span class="n">snapshots</span> <span class="o">=</span> <span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="s1">'Status'</span><span class="p">],</span> <span class="n">snapshots</span><span class="p">)</span>
+ <span class="n">snapshots</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="s1">'SnapshotCreateTime'</span><span class="p">],</span> <span class="n">reverse</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">snapshots</span></div>
+
+<div class="viewcode-block" id="RedshiftHook.restore_from_cluster_snapshot"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.redshift_hook.RedshiftHook.restore_from_cluster_snapshot">[docs]</a> <span class="k">def</span> <span class="nf">restore_from_cluster_snapshot</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cluster_identifier</span><span class="p">,</span> <span class="n">snapshot_identifier</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Restores a cluster from its snapshot</span>
+
+<span class="sd"> :param cluster_identifier: unique identifier of a cluster</span>
+<span class="sd"> :type cluster_identifier: str</span>
+<span class="sd"> :param snapshot_identifier: unique identifier for a snapshot of a cluster</span>
+<span class="sd"> :type snapshot_identifier: str</span>
+<span class="sd"> """</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">restore_from_cluster_snapshot</span><span class="p">(</span>
+ <span class="n">ClusterIdentifier</span><span class="o">=</span><span class="n">cluster_identifier</span><span class="p">,</span>
+ <span class="n">SnapshotIdentifier</span><span class="o">=</span><span class="n">snapshot_identifier</span>
+ <span class="p">)</span>
+ <span class="k">return</span> <span class="n">response</span><span class="p">[</span><span class="s1">'Cluster'</span><span class="p">]</span> <span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s1">'Cluster'</span><span class="p">]</span> <span class="k">else</span> <span class="kc">None</span></div>
+
+<div class="viewcode-block" id="RedshiftHook.create_cluster_snapshot"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.redshift_hook.RedshiftHook.create_cluster_snapshot">[docs]</a> <span class="k">def</span> <span class="nf">create_cluster_snapshot</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">snapshot_identifier</span><span class="p">,</span> <span class="n">cluster_identifier</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Creates a snapshot of a cluster</span>
+
+<span class="sd"> :param snapshot_identifier: unique identifier for a snapshot of a cluster</span>
+<span class="sd"> :type snapshot_identifier: str</span>
+<span class="sd"> :param cluster_identifier: unique identifier of a cluster</span>
+<span class="sd"> :type cluster_identifier: str</span>
+<span class="sd"> """</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">create_cluster_snapshot</span><span class="p">(</span>
+ <span class="n">SnapshotIdentifier</span><span class="o">=</span><span class="n">snapshot_identifier</span><span class="p">,</span>
+ <span class="n">ClusterIdentifier</span><span class="o">=</span><span class="n">cluster_identifier</span><span class="p">,</span>
+ <span class="p">)</span>
+ <span class="k">return</span> <span class="n">response</span><span class="p">[</span><span class="s1">'Snapshot'</span><span class="p">]</span> <span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s1">'Snapshot'</span><span class="p">]</span> <span class="k">else</span> <span class="kc">None</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/sftp_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/sftp_hook.html b/_modules/airflow/contrib/hooks/sftp_hook.html
new file mode 100644
index 0000000..6b048cb
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/sftp_hook.html
@@ -0,0 +1,404 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.sftp_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.sftp_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.sftp_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">stat</span>
+<span class="kn">import</span> <span class="nn">pysftp</span>
+<span class="kn">import</span> <span class="nn">logging</span>
+<span class="kn">import</span> <span class="nn">datetime</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+
+
+<div class="viewcode-block" id="SFTPHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sftp_hook.SFTPHook">[docs]</a><span class="k">class</span> <span class="nc">SFTPHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with SFTP. Aims to be interchangeable with FTPHook.</span>
+
+<span class="sd"> Pitfalls: - In contrast with FTPHook describe_directory only returns size, type and</span>
+<span class="sd"> modify. It doesn't return unix.owner, unix.mode, perm, unix.group and</span>
+<span class="sd"> unique.</span>
+<span class="sd"> - retrieve_file and store_file only take a local full path and not a</span>
+<span class="sd"> buffer.</span>
+<span class="sd"> - If no mode is passed to create_directory it will be created with 777</span>
+<span class="sd"> permissions.</span>
+
+<span class="sd"> Errors that may occur throughout but should be handled downstream.</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ftp_conn_id</span><span class="o">=</span><span class="s1">'sftp_default'</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">ftp_conn_id</span> <span class="o">=</span> <span class="n">ftp_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="kc">None</span>
+
+<div class="viewcode-block" id="SFTPHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sftp_hook.SFTPHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns an SFTP connection object</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">params</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ftp_conn_id</span><span class="p">)</span>
+ <span class="n">cnopts</span> <span class="o">=</span> <span class="n">pysftp</span><span class="o">.</span><span class="n">CnOpts</span><span class="p">()</span>
+ <span class="k">if</span> <span class="p">(</span><span class="s1">'ignore_hostkey_verification'</span> <span class="ow">in</span> <span class="n">params</span><span class="o">.</span><span class="n">extra_dejson</span> <span class="ow">and</span>
+ <span class="n">params</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s1">'ignore_hostkey_verification'</span><span class="p">]):</span>
+ <span class="n">cnopts</span><span class="o">.</span><span class="n">hostkeys</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="n">conn_params</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'host'</span><span class="p">:</span> <span class="n">params</span><span class="o">.</span><span class="n">host</span><span class="p">,</span>
+ <span class="s1">'port'</span><span class="p">:</span> <span class="n">params</span><span class="o">.</span><span class="n">port</span><span class="p">,</span>
+ <span class="s1">'username'</span><span class="p">:</span> <span class="n">params</span><span class="o">.</span><span class="n">login</span><span class="p">,</span>
+ <span class="s1">'cnopts'</span><span class="p">:</span> <span class="n">cnopts</span>
+ <span class="p">}</span>
+ <span class="k">if</span> <span class="n">params</span><span class="o">.</span><span class="n">password</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">conn_params</span><span class="p">[</span><span class="s1">'password'</span><span class="p">]</span> <span class="o">=</span> <span class="n">params</span><span class="o">.</span><span class="n">password</span>
+ <span class="k">if</span> <span class="s1">'private_key'</span> <span class="ow">in</span> <span class="n">params</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">:</span>
+ <span class="n">conn_params</span><span class="p">[</span><span class="s1">'private_key'</span><span class="p">]</span> <span class="o">=</span> <span class="n">params</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s1">'private_key'</span><span class="p">]</span>
+ <span class="k">if</span> <span class="s1">'private_key_pass'</span> <span class="ow">in</span> <span class="n">params</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">:</span>
+ <span class="n">conn_params</span><span class="p">[</span><span class="s1">'private_key_pass'</span><span class="p">]</span> <span class="o">=</span> <span class="n">params</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s1">'private_key_pass'</span><span class="p">]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="n">pysftp</span><span class="o">.</span><span class="n">Connection</span><span class="p">(</span><span class="o">**</span><span class="n">conn_params</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span></div>
+
+<div class="viewcode-block" id="SFTPHook.close_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sftp_hook.SFTPHook.close_conn">[docs]</a> <span class="k">def</span> <span class="nf">close_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Closes the connection. An error will occur if the</span>
+<span class="sd"> connection wasnt ever opened.</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="kc">None</span></div>
+
+<div class="viewcode-block" id="SFTPHook.describe_directory"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sftp_hook.SFTPHook.describe_directory">[docs]</a> <span class="k">def</span> <span class="nf">describe_directory</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns a dictionary of {filename: {attributes}} for all files</span>
+<span class="sd"> on the remote system (where the MLSD command is supported).</span>
+<span class="sd"> :param path: full path to the remote directory</span>
+<span class="sd"> :type path: str</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">flist</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">listdir_attr</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
+ <span class="n">files</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">flist</span><span class="p">:</span>
+ <span class="n">modify</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span>
+ <span class="n">f</span><span class="o">.</span><span class="n">st_mtime</span><span class="p">)</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s1">'%Y%m</span><span class="si">%d</span><span class="s1">%H%M%S'</span><span class="p">)</span>
+ <span class="n">files</span><span class="p">[</span><span class="n">f</span><span class="o">.</span><span class="n">filename</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'size'</span><span class="p">:</span> <span class="n">f</span><span class="o">.</span><span class="n">st_size</span><span class="p">,</span>
+ <span class="s1">'type'</span><span class="p">:</span> <span class="s1">'dir'</span> <span class="k">if</span> <span class="n">stat</span><span class="o">.</span><span class="n">S_ISDIR</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">st_mode</span><span class="p">)</span> <span class="k">else</span> <span class="s1">'file'</span><span class="p">,</span>
+ <span class="s1">'modify'</span><span class="p">:</span> <span class="n">modify</span><span class="p">}</span>
+ <span class="k">return</span> <span class="n">files</span></div>
+
+<div class="viewcode-block" id="SFTPHook.list_directory"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sftp_hook.SFTPHook.list_directory">[docs]</a> <span class="k">def</span> <span class="nf">list_directory</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns a list of files on the remote system.</span>
+<span class="sd"> :param path: full path to the remote directory to list</span>
+<span class="sd"> :type path: str</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">files</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">listdir</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">files</span></div>
+
+<div class="viewcode-block" id="SFTPHook.create_directory"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sftp_hook.SFTPHook.create_directory">[docs]</a> <span class="k">def</span> <span class="nf">create_directory</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="mi">777</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Creates a directory on the remote system.</span>
+<span class="sd"> :param path: full path to the remote directory to create</span>
+<span class="sd"> :type path: str</span>
+<span class="sd"> :param mode: int representation of octal mode for directory</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">mode</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="SFTPHook.delete_directory"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sftp_hook.SFTPHook.delete_directory">[docs]</a> <span class="k">def</span> <span class="nf">delete_directory</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Deletes a directory on the remote system.</span>
+<span class="sd"> :param path: full path to the remote directory to delete</span>
+<span class="sd"> :type path: str</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">rmdir</span><span class="p">(</span><span class="n">path</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="SFTPHook.retrieve_file"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sftp_hook.SFTPHook.retrieve_file">[docs]</a> <span class="k">def</span> <span class="nf">retrieve_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">remote_full_path</span><span class="p">,</span> <span class="n">local_full_path</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Transfers the remote file to a local location.</span>
+<span class="sd"> If local_full_path is a string path, the file will be put</span>
+<span class="sd"> at that location</span>
+<span class="sd"> :param remote_full_path: full path to the remote file</span>
+<span class="sd"> :type remote_full_path: str</span>
+<span class="sd"> :param local_full_path: full path to the local file</span>
+<span class="sd"> :type local_full_path: str</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Retrieving file from FTP: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">remote_full_path</span><span class="p">))</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">remote_full_path</span><span class="p">,</span> <span class="n">local_full_path</span><span class="p">)</span>
+ <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Finished retrieving file from FTP: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">remote_full_path</span><span class="p">))</span></div>
+
+<div class="viewcode-block" id="SFTPHook.store_file"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sftp_hook.SFTPHook.store_file">[docs]</a> <span class="k">def</span> <span class="nf">store_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">remote_full_path</span><span class="p">,</span> <span class="n">local_full_path</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Transfers a local file to the remote location.</span>
+<span class="sd"> If local_full_path_or_buffer is a string path, the file will be read</span>
+<span class="sd"> from that location</span>
+<span class="sd"> :param remote_full_path: full path to the remote file</span>
+<span class="sd"> :type remote_full_path: str</span>
+<span class="sd"> :param local_full_path: full path to the local file</span>
+<span class="sd"> :type local_full_path: str</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="n">local_full_path</span><span class="p">,</span> <span class="n">remote_full_path</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="SFTPHook.delete_file"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sftp_hook.SFTPHook.delete_file">[docs]</a> <span class="k">def</span> <span class="nf">delete_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Removes a file on the FTP Server</span>
+<span class="sd"> :param path: full path to the remote file</span>
+<span class="sd"> :type path: str</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">path</span><span class="p">)</span></div>
+
+ <span class="k">def</span> <span class="nf">get_mod_time</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">ftp_mdtm</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">path</span><span class="p">)</span><span class="o">.</span><span class="n">st_mtime</span>
+ <span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">ftp_mdtm</span><span class="p">)</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s1">'%Y%m</span><span class="si">%d</span><span class="s1">%H%M%S'</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/slack_webhook_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/slack_webhook_hook.html b/_modules/airflow/contrib/hooks/slack_webhook_hook.html
new file mode 100644
index 0000000..e3e9dfc
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/slack_webhook_hook.html
@@ -0,0 +1,364 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.slack_webhook_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.slack_webhook_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.slack_webhook_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+<span class="kn">import</span> <span class="nn">json</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.http_hook</span> <span class="k">import</span> <span class="n">HttpHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+
+
+<div class="viewcode-block" id="SlackWebhookHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.slack_webhook_hook.SlackWebhookHook">[docs]</a><span class="k">class</span> <span class="nc">SlackWebhookHook</span><span class="p">(</span><span class="n">HttpHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> This hook allows you to post messages to Slack using incoming webhooks.</span>
+<span class="sd"> Takes both Slack webhook token directly and connection that has Slack webhook token.</span>
+<span class="sd"> If both supplied, Slack webhook token will be used.</span>
+
+<span class="sd"> Each Slack webhook token can be pre-configured to use a specific channel, username and</span>
+<span class="sd"> icon. You can override these defaults in this hook.</span>
+
+<span class="sd"> :param http_conn_id: connection that has Slack webhook token in the extra field</span>
+<span class="sd"> :type http_conn_id: str</span>
+<span class="sd"> :param webhook_token: Slack webhook token</span>
+<span class="sd"> :type webhook_token: str</span>
+<span class="sd"> :param message: The message you want to send on Slack</span>
+<span class="sd"> :type message: str</span>
+<span class="sd"> :param channel: The channel the message should be posted to</span>
+<span class="sd"> :type channel: str</span>
+<span class="sd"> :param username: The username to post to slack with</span>
+<span class="sd"> :type username: str</span>
+<span class="sd"> :param icon_emoji: The emoji to use as icon for the user posting to Slack</span>
+<span class="sd"> :type icon_emoji: str</span>
+<span class="sd"> :param link_names: Whether or not to find and link channel and usernames in your</span>
+<span class="sd"> message</span>
+<span class="sd"> :type link_names: bool</span>
+<span class="sd"> :param proxy: Proxy to use to make the Slack webhook call</span>
+<span class="sd"> :type proxy: str</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">http_conn_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">webhook_token</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">message</span><span class="o">=</span><span class="s2">""</span><span class="p">,</span>
+ <span class="n">channel</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">username</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">icon_emoji</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">link_names</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">proxy</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span>
+ <span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">SlackWebhookHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">http_conn_id</span> <span class="o">=</span> <span class="n">http_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">webhook_token</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_token</span><span class="p">(</span><span class="n">webhook_token</span><span class="p">,</span> <span class="n">http_conn_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">message</span> <span class="o">=</span> <span class="n">message</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">channel</span> <span class="o">=</span> <span class="n">channel</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="o">=</span> <span class="n">username</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">icon_emoji</span> <span class="o">=</span> <span class="n">icon_emoji</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">link_names</span> <span class="o">=</span> <span class="n">link_names</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">proxy</span> <span class="o">=</span> <span class="n">proxy</span>
+
+ <span class="k">def</span> <span class="nf">_get_token</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">token</span><span class="p">,</span> <span class="n">http_conn_id</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Given either a manually set token or a conn_id, return the webhook_token to use</span>
+<span class="sd"> :param token: The manually provided token</span>
+<span class="sd"> :param conn_id: The conn_id provided</span>
+<span class="sd"> :return: webhook_token (str) to use</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">token</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">token</span>
+ <span class="k">elif</span> <span class="n">http_conn_id</span><span class="p">:</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">http_conn_id</span><span class="p">)</span>
+ <span class="n">extra</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span>
+ <span class="k">return</span> <span class="n">extra</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'webhook_token'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Cannot get token: No valid Slack '</span>
+ <span class="s1">'webhook token nor conn_id supplied'</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_build_slack_message</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Construct the Slack message. All relevant parameters are combined here to a valid</span>
+<span class="sd"> Slack json message</span>
+<span class="sd"> :return: Slack message (str) to send</span>
+<span class="sd"> """</span>
+ <span class="n">cmd</span> <span class="o">=</span> <span class="p">{}</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">channel</span><span class="p">:</span>
+ <span class="n">cmd</span><span class="p">[</span><span class="s1">'channel'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">channel</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">:</span>
+ <span class="n">cmd</span><span class="p">[</span><span class="s1">'username'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">username</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">icon_emoji</span><span class="p">:</span>
+ <span class="n">cmd</span><span class="p">[</span><span class="s1">'icon_emoji'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">icon_emoji</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">link_names</span><span class="p">:</span>
+ <span class="n">cmd</span><span class="p">[</span><span class="s1">'link_names'</span><span class="p">]</span> <span class="o">=</span> <span class="mi">1</span>
+
+ <span class="c1"># there should always be a message to post ;-)</span>
+ <span class="n">cmd</span><span class="p">[</span><span class="s1">'text'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">message</span>
+ <span class="k">return</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">cmd</span><span class="p">)</span>
+
+<div class="viewcode-block" id="SlackWebhookHook.execute"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.slack_webhook_hook.SlackWebhookHook.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Remote Popen (actually execute the slack webhook call)</span>
+
+<span class="sd"> :param cmd: command to remotely execute</span>
+<span class="sd"> :param kwargs: extra arguments to Popen (see subprocess.Popen)</span>
+<span class="sd"> """</span>
+ <span class="n">proxies</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">proxy</span><span class="p">:</span>
+ <span class="c1"># we only need https proxy for Slack, as the endpoint is https</span>
+ <span class="n">proxies</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'https'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">proxy</span><span class="p">}</span>
+
+ <span class="n">slack_message</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_slack_message</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">endpoint</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">webhook_token</span><span class="p">,</span>
+ <span class="n">data</span><span class="o">=</span><span class="n">slack_message</span><span class="p">,</span>
+ <span class="n">headers</span><span class="o">=</span><span class="p">{</span><span class="s1">'Content-type'</span><span class="p">:</span> <span class="s1">'application/json'</span><span class="p">},</span>
+ <span class="n">extra_options</span><span class="o">=</span><span class="p">{</span><span class="s1">'proxies'</span><span class="p">:</span> <span class="n">proxies</span><span class="p">})</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[46/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/gcp_api_base_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/gcp_api_base_hook.html b/_modules/airflow/contrib/hooks/gcp_api_base_hook.html
new file mode 100644
index 0000000..c81584e
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/gcp_api_base_hook.html
@@ -0,0 +1,379 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.gcp_api_base_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.gcp_api_base_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.gcp_api_base_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+<span class="kn">import</span> <span class="nn">json</span>
+
+<span class="kn">import</span> <span class="nn">httplib2</span>
+<span class="kn">from</span> <span class="nn">oauth2client.client</span> <span class="k">import</span> <span class="n">GoogleCredentials</span>
+<span class="kn">from</span> <span class="nn">oauth2client.service_account</span> <span class="k">import</span> <span class="n">ServiceAccountCredentials</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+
+<div class="viewcode-block" id="GoogleCloudBaseHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook">[docs]</a><span class="k">class</span> <span class="nc">GoogleCloudBaseHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> A base hook for Google cloud-related hooks. Google cloud has a shared REST</span>
+<span class="sd"> API client that is built in the same way no matter which service you use.</span>
+<span class="sd"> This class helps construct and authorize the credentials needed to then</span>
+<span class="sd"> call apiclient.discovery.build() to actually discover and build a client</span>
+<span class="sd"> for a Google cloud service.</span>
+
+<span class="sd"> The class also contains some miscellaneous helper functions.</span>
+
+<span class="sd"> All hook derived from this base hook use the 'Google Cloud Platform' connection</span>
+<span class="sd"> type. Two ways of authentication are supported:</span>
+
+<span class="sd"> Default credentials: Only specify 'Project Id'. Then you need to have executed</span>
+<span class="sd"> ``gcloud auth`` on the Airflow worker machine.</span>
+
+<span class="sd"> JSON key file: Specify 'Project Id', 'Key Path' and 'Scope'.</span>
+
+<span class="sd"> Legacy P12 key files are not supported.</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span> <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> :param gcp_conn_id: The connection ID to use when fetching connection info.</span>
+<span class="sd"> :type gcp_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request must have</span>
+<span class="sd"> domain-wide delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">extras</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">extra_dejson</span>
+
+ <span class="k">def</span> <span class="nf">_get_credentials</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns the Credentials object for Google API</span>
+<span class="sd"> """</span>
+ <span class="n">key_path</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_field</span><span class="p">(</span><span class="s1">'key_path'</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
+ <span class="n">keyfile_dict</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_field</span><span class="p">(</span><span class="s1">'keyfile_dict'</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
+ <span class="n">scope</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_field</span><span class="p">(</span><span class="s1">'scope'</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">key_path</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">keyfile_dict</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Getting connection using `gcloud auth` user, '</span>
+ <span class="s1">'since no key file is defined for hook.'</span><span class="p">)</span>
+ <span class="n">credentials</span> <span class="o">=</span> <span class="n">GoogleCredentials</span><span class="o">.</span><span class="n">get_application_default</span><span class="p">()</span>
+ <span class="k">elif</span> <span class="n">key_path</span><span class="p">:</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">scope</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Scope should be defined when using a key file.'</span><span class="p">)</span>
+ <span class="n">scopes</span> <span class="o">=</span> <span class="p">[</span><span class="n">s</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">scope</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">','</span><span class="p">)]</span>
+
+ <span class="c1"># Get credentials from a JSON file.</span>
+ <span class="k">if</span> <span class="n">key_path</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s1">'.json'</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Getting connection using a JSON key file.'</span><span class="p">)</span>
+ <span class="n">credentials</span> <span class="o">=</span> <span class="n">ServiceAccountCredentials</span> \
+ <span class="o">.</span><span class="n">from_json_keyfile_name</span><span class="p">(</span><span class="n">key_path</span><span class="p">,</span> <span class="n">scopes</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="n">key_path</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s1">'.p12'</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Legacy P12 key file are not supported, '</span>
+ <span class="s1">'use a JSON key file.'</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Unrecognised extension for key file.'</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">scope</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Scope should be defined when using key JSON.'</span><span class="p">)</span>
+ <span class="n">scopes</span> <span class="o">=</span> <span class="p">[</span><span class="n">s</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">scope</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">','</span><span class="p">)]</span>
+
+ <span class="c1"># Get credentials from JSON data provided in the UI.</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">keyfile_dict</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">keyfile_dict</span><span class="p">)</span>
+
+ <span class="c1"># Depending on how the JSON was formatted, it may contain</span>
+ <span class="c1"># escaped newlines. Convert those to actual newlines.</span>
+ <span class="n">keyfile_dict</span><span class="p">[</span><span class="s1">'private_key'</span><span class="p">]</span> <span class="o">=</span> <span class="n">keyfile_dict</span><span class="p">[</span><span class="s1">'private_key'</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span>
+ <span class="s1">'</span><span class="se">\\</span><span class="s1">n'</span><span class="p">,</span> <span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">)</span>
+
+ <span class="n">credentials</span> <span class="o">=</span> <span class="n">ServiceAccountCredentials</span> \
+ <span class="o">.</span><span class="n">from_json_keyfile_dict</span><span class="p">(</span><span class="n">keyfile_dict</span><span class="p">,</span> <span class="n">scopes</span><span class="p">)</span>
+ <span class="k">except</span> <span class="n">json</span><span class="o">.</span><span class="n">decoder</span><span class="o">.</span><span class="n">JSONDecodeError</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Invalid key JSON.'</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">credentials</span><span class="o">.</span><span class="n">create_delegated</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span> \
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="k">else</span> <span class="n">credentials</span>
+
+ <span class="k">def</span> <span class="nf">_get_access_token</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns a valid access token from Google API Credentials</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_credentials</span><span class="p">()</span><span class="o">.</span><span class="n">get_access_token</span><span class="p">()</span><span class="o">.</span><span class="n">access_token</span>
+
+ <span class="k">def</span> <span class="nf">_authorize</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns an authorized HTTP object to be used to build a Google cloud</span>
+<span class="sd"> service hook connection.</span>
+<span class="sd"> """</span>
+ <span class="n">credentials</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_credentials</span><span class="p">()</span>
+ <span class="n">http</span> <span class="o">=</span> <span class="n">httplib2</span><span class="o">.</span><span class="n">Http</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">credentials</span><span class="o">.</span><span class="n">authorize</span><span class="p">(</span><span class="n">http</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_get_field</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">f</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Fetches a field from extras, and returns it. This is some Airflow</span>
+<span class="sd"> magic. The google_cloud_platform hook type adds custom UI elements</span>
+<span class="sd"> to the hook page, which allow admins to specify service_account,</span>
+<span class="sd"> key_path, etc. They get formatted as shown below.</span>
+<span class="sd"> """</span>
+ <span class="n">long_f</span> <span class="o">=</span> <span class="s1">'extra__google_cloud_platform__</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">long_f</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">extras</span><span class="p">:</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">extras</span><span class="p">[</span><span class="n">long_f</span><span class="p">]</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">default</span>
+
+ <span class="nd">@property</span>
+ <span class="k">def</span> <span class="nf">project_id</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_field</span><span class="p">(</span><span class="s1">'project'</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/gcp_dataflow_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/gcp_dataflow_hook.html b/_modules/airflow/contrib/hooks/gcp_dataflow_hook.html
index 82e3ccd..ce6f6ff 100644
--- a/_modules/airflow/contrib/hooks/gcp_dataflow_hook.html
+++ b/_modules/airflow/contrib/hooks/gcp_dataflow_hook.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,23 @@
<h1>Source code for airflow.contrib.hooks.gcp_dataflow_hook</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">import</span> <span class="nn">json</span>
<span class="kn">import</span> <span class="nn">select</span>
<span class="kn">import</span> <span class="nn">subprocess</span>
<span class="kn">import</span> <span class="nn">time</span>
@@ -190,18 +198,25 @@
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="k">import</span> <span class="n">GoogleCloudBaseHook</span>
<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+<span class="c1"># This is the default location</span>
+<span class="c1"># https://cloud.google.com/dataflow/pipelines/specifying-exec-params</span>
+<span class="n">DEFAULT_DATAFLOW_LOCATION</span> <span class="o">=</span> <span class="s1">'us-central1'</span>
+
<span class="k">class</span> <span class="nc">_DataflowJob</span><span class="p">(</span><span class="n">LoggingMixin</span><span class="p">):</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">,</span> <span class="n">project_number</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">,</span> <span class="n">project_number</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">location</span><span class="p">,</span> <span class="n">poll_sleep</span><span class="o">=</span><span class="mi">10</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_dataflow</span> <span class="o">=</span> <span class="n">dataflow</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_project_number</span> <span class="o">=</span> <span class="n">project_number</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_job_name</span> <span class="o">=</span> <span class="n">name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_job_location</span> <span class="o">=</span> <span class="n">location</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_job_id</span> <span class="o">=</span> <span class="kc">None</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_job</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_job</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_poll_sleep</span> <span class="o">=</span> <span class="n">poll_sleep</span>
<span class="k">def</span> <span class="nf">_get_job_id_from_name</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="n">jobs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_dataflow</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span><span class="o">.</span><span class="n">list</span><span class="p">(</span>
- <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_project_number</span>
+ <span class="n">jobs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_dataflow</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">locations</span><span class="p">()</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span><span class="o">.</span><span class="n">list</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_project_number</span><span class="p">,</span>
+ <span class="n">location</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_job_location</span>
<span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
<span class="k">for</span> <span class="n">job</span> <span class="ow">in</span> <span class="n">jobs</span><span class="p">[</span><span class="s1">'jobs'</span><span class="p">]:</span>
<span class="k">if</span> <span class="n">job</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job_name</span><span class="p">:</span>
@@ -210,28 +225,39 @@
<span class="k">return</span> <span class="kc">None</span>
<span class="k">def</span> <span class="nf">_get_job</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job_id</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job_name</span><span class="p">:</span>
<span class="n">job</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_job_id_from_name</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
- <span class="n">job</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_dataflow</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_project_number</span><span class="p">,</span>
- <span class="n">jobId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_job_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
- <span class="k">if</span> <span class="s1">'currentState'</span> <span class="ow">in</span> <span class="n">job</span><span class="p">:</span>
+ <span class="n">job</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_dataflow</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_project_number</span><span class="p">,</span>
+ <span class="n">jobId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_job_id</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+
+ <span class="k">if</span> <span class="n">job</span> <span class="ow">and</span> <span class="s1">'currentState'</span> <span class="ow">in</span> <span class="n">job</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
<span class="s1">'Google Cloud DataFlow job </span><span class="si">%s</span><span class="s1"> is </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
<span class="n">job</span><span class="p">[</span><span class="s1">'name'</span><span class="p">],</span> <span class="n">job</span><span class="p">[</span><span class="s1">'currentState'</span><span class="p">]</span>
<span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
+ <span class="k">elif</span> <span class="n">job</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
<span class="s1">'Google Cloud DataFlow with job_id </span><span class="si">%s</span><span class="s1"> has name </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_job_id</span><span class="p">,</span> <span class="n">job</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]</span>
<span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'Google Cloud DataFlow job not available yet..'</span>
+ <span class="p">)</span>
+
<span class="k">return</span> <span class="n">job</span>
<span class="k">def</span> <span class="nf">wait_for_done</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
- <span class="k">if</span> <span class="s1">'currentState'</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job</span><span class="p">:</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job</span> <span class="ow">and</span> <span class="s1">'currentState'</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job</span><span class="p">:</span>
<span class="k">if</span> <span class="s1">'JOB_STATE_DONE'</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job</span><span class="p">[</span><span class="s1">'currentState'</span><span class="p">]:</span>
<span class="k">return</span> <span class="kc">True</span>
+ <span class="k">elif</span> <span class="s1">'JOB_STATE_RUNNING'</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job</span><span class="p">[</span><span class="s1">'currentState'</span><span class="p">]</span> <span class="ow">and</span> \
+ <span class="s1">'JOB_TYPE_STREAMING'</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job</span><span class="p">[</span><span class="s1">'type'</span><span class="p">]:</span>
+ <span class="k">return</span> <span class="kc">True</span>
<span class="k">elif</span> <span class="s1">'JOB_STATE_FAILED'</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job</span><span class="p">[</span><span class="s1">'currentState'</span><span class="p">]:</span>
<span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s2">"Google Cloud Dataflow job </span><span class="si">{}</span><span class="s2"> has failed."</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_job</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]))</span>
@@ -239,7 +265,7 @@
<span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s2">"Google Cloud Dataflow job </span><span class="si">{}</span><span class="s2"> was cancelled."</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_job</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]))</span>
<span class="k">elif</span> <span class="s1">'JOB_STATE_RUNNING'</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job</span><span class="p">[</span><span class="s1">'currentState'</span><span class="p">]:</span>
- <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">10</span><span class="p">)</span>
+ <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_poll_sleep</span><span class="p">)</span>
<span class="k">elif</span> <span class="s1">'JOB_STATE_PENDING'</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job</span><span class="p">[</span><span class="s1">'currentState'</span><span class="p">]:</span>
<span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">15</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
@@ -258,13 +284,21 @@
<span class="k">class</span> <span class="nc">_Dataflow</span><span class="p">(</span><span class="n">LoggingMixin</span><span class="p">):</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cmd</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">Popen</span><span class="p">(</span><span class="n">cmd</span><span class="p">,</span> <span class="n">shell</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">stdout</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span>
- <span class="n">stderr</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Running command: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="s1">' '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">cmd</span><span class="p">))</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">Popen</span><span class="p">(</span>
+ <span class="n">cmd</span><span class="p">,</span>
+ <span class="n">shell</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">stdout</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span>
+ <span class="n">stderr</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span>
+ <span class="n">close_fds</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">_line</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">fd</span><span class="p">):</span>
<span class="k">if</span> <span class="n">fd</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">stderr</span><span class="o">.</span><span class="n">fileno</span><span class="p">():</span>
- <span class="n">line</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">stderr</span><span class="o">.</span><span class="n">readline</span><span class="p">()</span>
- <span class="k">return</span> <span class="n">line</span>
+ <span class="n">lines</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">stderr</span><span class="o">.</span><span class="n">readlines</span><span class="p">()</span>
+ <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">lines</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">line</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
+ <span class="k">if</span> <span class="n">lines</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">lines</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
<span class="k">if</span> <span class="n">fd</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">fileno</span><span class="p">():</span>
<span class="n">line</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">readline</span><span class="p">()</span>
<span class="k">return</span> <span class="n">line</span>
@@ -283,7 +317,8 @@
<span class="k">if</span> <span class="n">ret</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">for</span> <span class="n">fd</span> <span class="ow">in</span> <span class="n">ret</span><span class="p">[</span><span class="mi">0</span><span class="p">]:</span>
<span class="n">line</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_line</span><span class="p">(</span><span class="n">fd</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="n">line</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
+ <span class="k">if</span> <span class="n">line</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="n">line</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Waiting for DataFlow process to complete."</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">returncode</span> <span class="ow">is</span> <span class="ow">not</span> <span class="mi">0</span><span class="p">:</span>
@@ -295,7 +330,9 @@
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
<span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
- <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">poll_sleep</span><span class="o">=</span><span class="mi">10</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">poll_sleep</span> <span class="o">=</span> <span class="n">poll_sleep</span>
<span class="nb">super</span><span class="p">(</span><span class="n">DataFlowHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="p">)</span>
<div class="viewcode-block" id="DataFlowHook.get_conn"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_dataflow_hook.DataFlowHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
@@ -305,30 +342,95 @@
<span class="n">http_authorized</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
<span class="k">return</span> <span class="n">build</span><span class="p">(</span><span class="s1">'dataflow'</span><span class="p">,</span> <span class="s1">'v1b3'</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">)</span></div>
- <span class="k">def</span> <span class="nf">_start_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">command_prefix</span><span class="p">):</span>
- <span class="n">cmd</span> <span class="o">=</span> <span class="n">command_prefix</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_cmd</span><span class="p">(</span><span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">)</span>
+ <span class="k">def</span> <span class="nf">_start_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span>
+ <span class="n">command_prefix</span><span class="p">,</span> <span class="n">label_formatter</span><span class="p">):</span>
+ <span class="n">variables</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_set_variables</span><span class="p">(</span><span class="n">variables</span><span class="p">)</span>
+ <span class="n">cmd</span> <span class="o">=</span> <span class="n">command_prefix</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_cmd</span><span class="p">(</span><span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span>
+ <span class="n">label_formatter</span><span class="p">)</span>
<span class="n">_Dataflow</span><span class="p">(</span><span class="n">cmd</span><span class="p">)</span><span class="o">.</span><span class="n">wait_for_done</span><span class="p">()</span>
- <span class="n">_DataflowJob</span><span class="p">(</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">(),</span> <span class="n">variables</span><span class="p">[</span><span class="s1">'project'</span><span class="p">],</span> <span class="n">name</span><span class="p">)</span><span class="o">.</span><span class="n">wait_for_done</span><span class="p">()</span>
+ <span class="n">_DataflowJob</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">(),</span> <span class="n">variables</span><span class="p">[</span><span class="s1">'project'</span><span class="p">],</span> <span class="n">name</span><span class="p">,</span>
+ <span class="n">variables</span><span class="p">[</span><span class="s1">'region'</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">poll_sleep</span><span class="p">)</span><span class="o">.</span><span class="n">wait_for_done</span><span class="p">()</span>
- <span class="k">def</span> <span class="nf">start_java_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">):</span>
- <span class="n">name</span> <span class="o">=</span> <span class="n">task_id</span> <span class="o">+</span> <span class="s2">"-"</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">uuid</span><span class="o">.</span><span class="n">uuid1</span><span class="p">())[:</span><span class="mi">8</span><span class="p">]</span>
+ <span class="nd">@staticmethod</span>
+ <span class="k">def</span> <span class="nf">_set_variables</span><span class="p">(</span><span class="n">variables</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">variables</span><span class="p">[</span><span class="s1">'project'</span><span class="p">]</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s1">'Project not specified'</span><span class="p">)</span>
+ <span class="k">if</span> <span class="s1">'region'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">variables</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
+ <span class="n">variables</span><span class="p">[</span><span class="s1">'region'</span><span class="p">]</span> <span class="o">=</span> <span class="n">DEFAULT_DATAFLOW_LOCATION</span>
+ <span class="k">return</span> <span class="n">variables</span>
+
+ <span class="k">def</span> <span class="nf">start_java_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">,</span> <span class="n">job_class</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">append_job_name</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">append_job_name</span><span class="p">:</span>
+ <span class="n">name</span> <span class="o">=</span> <span class="n">task_id</span> <span class="o">+</span> <span class="s2">"-"</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">uuid</span><span class="o">.</span><span class="n">uuid1</span><span class="p">())[:</span><span class="mi">8</span><span class="p">]</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">name</span> <span class="o">=</span> <span class="n">task_id</span>
<span class="n">variables</span><span class="p">[</span><span class="s1">'jobName'</span><span class="p">]</span> <span class="o">=</span> <span class="n">name</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">_start_dataflow</span><span class="p">(</span>
- <span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="p">[</span><span class="s2">"java"</span><span class="p">,</span> <span class="s2">"-jar"</span><span class="p">])</span>
- <span class="k">def</span> <span class="nf">start_python_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">,</span> <span class="n">py_options</span><span class="p">):</span>
- <span class="n">name</span> <span class="o">=</span> <span class="n">task_id</span> <span class="o">+</span> <span class="s2">"-"</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">uuid</span><span class="o">.</span><span class="n">uuid1</span><span class="p">())[:</span><span class="mi">8</span><span class="p">]</span>
- <span class="n">variables</span><span class="p">[</span><span class="s2">"job_name"</span><span class="p">]</span> <span class="o">=</span> <span class="n">name</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">_start_dataflow</span><span class="p">(</span>
- <span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="p">[</span><span class="s2">"python"</span><span class="p">]</span> <span class="o">+</span> <span class="n">py_options</span><span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">_build_cmd</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">):</span>
- <span class="n">command</span> <span class="o">=</span> <span class="p">[</span><span class="n">dataflow</span><span class="p">,</span> <span class="s2">"--runner=DataflowRunner"</span><span class="p">]</span>
+ <span class="k">def</span> <span class="nf">label_formatter</span><span class="p">(</span><span class="n">labels_dict</span><span class="p">):</span>
+ <span class="k">return</span> <span class="p">[</span><span class="s1">'--labels=</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">labels_dict</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">' '</span><span class="p">,</span> <span class="s1">''</span><span class="p">))]</span>
+ <span class="n">command_prefix</span> <span class="o">=</span> <span class="p">([</span><span class="s2">"java"</span><span class="p">,</span> <span class="s2">"-cp"</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">,</span> <span class="n">job_class</span><span class="p">]</span> <span class="k">if</span> <span class="n">job_class</span>
+ <span class="k">else</span> <span class="p">[</span><span class="s2">"java"</span><span class="p">,</span> <span class="s2">"-jar"</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">])</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_start_dataflow</span><span class="p">(</span><span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span>
+ <span class="n">command_prefix</span><span class="p">,</span> <span class="n">label_formatter</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">start_template_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">parameters</span><span class="p">,</span> <span class="n">dataflow_template</span><span class="p">,</span>
+ <span class="n">append_job_name</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">append_job_name</span><span class="p">:</span>
+ <span class="n">name</span> <span class="o">=</span> <span class="n">task_id</span> <span class="o">+</span> <span class="s2">"-"</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">uuid</span><span class="o">.</span><span class="n">uuid1</span><span class="p">())[:</span><span class="mi">8</span><span class="p">]</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">name</span> <span class="o">=</span> <span class="n">task_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_start_template_dataflow</span><span class="p">(</span>
+ <span class="n">name</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">parameters</span><span class="p">,</span> <span class="n">dataflow_template</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">start_python_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">,</span> <span class="n">py_options</span><span class="p">,</span>
+ <span class="n">append_job_name</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">append_job_name</span><span class="p">:</span>
+ <span class="n">name</span> <span class="o">=</span> <span class="n">task_id</span> <span class="o">+</span> <span class="s2">"-"</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">uuid</span><span class="o">.</span><span class="n">uuid1</span><span class="p">())[:</span><span class="mi">8</span><span class="p">]</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">name</span> <span class="o">=</span> <span class="n">task_id</span>
+ <span class="n">variables</span><span class="p">[</span><span class="s1">'job_name'</span><span class="p">]</span> <span class="o">=</span> <span class="n">name</span>
+
+ <span class="k">def</span> <span class="nf">label_formatter</span><span class="p">(</span><span class="n">labels_dict</span><span class="p">):</span>
+ <span class="k">return</span> <span class="p">[</span><span class="s1">'--labels=</span><span class="si">{}</span><span class="s1">=</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>
+ <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">labels_dict</span><span class="o">.</span><span class="n">items</span><span class="p">()]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_start_dataflow</span><span class="p">(</span><span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span>
+ <span class="p">[</span><span class="s2">"python"</span><span class="p">]</span> <span class="o">+</span> <span class="n">py_options</span> <span class="o">+</span> <span class="p">[</span><span class="n">dataflow</span><span class="p">],</span>
+ <span class="n">label_formatter</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_build_cmd</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">label_formatter</span><span class="p">):</span>
+ <span class="n">command</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"--runner=DataflowRunner"</span><span class="p">]</span>
<span class="k">if</span> <span class="n">variables</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
- <span class="k">for</span> <span class="n">attr</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">variables</span><span class="o">.</span><span class="n">iteritems</span><span class="p">():</span>
- <span class="n">command</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"--"</span> <span class="o">+</span> <span class="n">attr</span> <span class="o">+</span> <span class="s2">"="</span> <span class="o">+</span> <span class="n">value</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">command</span></div>
+ <span class="k">for</span> <span class="n">attr</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">variables</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+ <span class="k">if</span> <span class="n">attr</span> <span class="o">==</span> <span class="s1">'labels'</span><span class="p">:</span>
+ <span class="n">command</span> <span class="o">+=</span> <span class="n">label_formatter</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="n">value</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">value</span><span class="o">.</span><span class="fm">__len__</span><span class="p">()</span> <span class="o"><</span> <span class="mi">1</span><span class="p">:</span>
+ <span class="n">command</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"--"</span> <span class="o">+</span> <span class="n">attr</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">command</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"--"</span> <span class="o">+</span> <span class="n">attr</span> <span class="o">+</span> <span class="s2">"="</span> <span class="o">+</span> <span class="n">value</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">command</span>
+
+ <span class="k">def</span> <span class="nf">_start_template_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">parameters</span><span class="p">,</span> <span class="n">dataflow_template</span><span class="p">):</span>
+ <span class="c1"># Builds RuntimeEnvironment from variables dictionary</span>
+ <span class="c1"># https://cloud.google.com/dataflow/docs/reference/rest/v1b3/RuntimeEnvironment</span>
+ <span class="n">environment</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'maxWorkers'</span><span class="p">,</span> <span class="s1">'zone'</span><span class="p">,</span> <span class="s1">'serviceAccountEmail'</span><span class="p">,</span> <span class="s1">'tempLocation'</span><span class="p">,</span>
+ <span class="s1">'bypassTempDirValidation'</span><span class="p">,</span> <span class="s1">'machineType'</span><span class="p">]:</span>
+ <span class="k">if</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">variables</span><span class="p">:</span>
+ <span class="n">environment</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="n">key</span><span class="p">:</span> <span class="n">variables</span><span class="p">[</span><span class="n">key</span><span class="p">]})</span>
+ <span class="n">body</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"jobName"</span><span class="p">:</span> <span class="n">name</span><span class="p">,</span>
+ <span class="s2">"parameters"</span><span class="p">:</span> <span class="n">parameters</span><span class="p">,</span>
+ <span class="s2">"environment"</span><span class="p">:</span> <span class="n">environment</span><span class="p">}</span>
+ <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">request</span> <span class="o">=</span> <span class="n">service</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">templates</span><span class="p">()</span><span class="o">.</span><span class="n">launch</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">variables</span><span class="p">[</span><span class="s1">'project'</span><span class="p">],</span>
+ <span class="n">gcsPath</span><span class="o">=</span><span class="n">dataflow_template</span><span class="p">,</span>
+ <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="n">variables</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_set_variables</span><span class="p">(</span><span class="n">variables</span><span class="p">)</span>
+ <span class="n">_DataflowJob</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">(),</span> <span class="n">variables</span><span class="p">[</span><span class="s1">'project'</span><span class="p">],</span> <span class="n">name</span><span class="p">,</span> <span class="n">variables</span><span class="p">[</span><span class="s1">'region'</span><span class="p">],</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">poll_sleep</span><span class="p">)</span><span class="o">.</span><span class="n">wait_for_done</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">response</span></div>
</pre></div>
</div>
[50/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html b/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html
new file mode 100644
index 0000000..68c9921
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html
@@ -0,0 +1,300 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.aws_dynamodb_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.aws_dynamodb_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.aws_dynamodb_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.aws_hook</span> <span class="k">import</span> <span class="n">AwsHook</span>
+
+
+<div class="viewcode-block" id="AwsDynamoDBHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_dynamodb_hook.AwsDynamoDBHook">[docs]</a><span class="k">class</span> <span class="nc">AwsDynamoDBHook</span><span class="p">(</span><span class="n">AwsHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with AWS DynamoDB.</span>
+
+<span class="sd"> :param table_keys: partition key and sort key</span>
+<span class="sd"> :type table_keys: list</span>
+<span class="sd"> :param table_name: target DynamoDB table</span>
+<span class="sd"> :type table_name: str</span>
+<span class="sd"> :param region_name: aws region name (example: us-east-1)</span>
+<span class="sd"> :type region_name: str</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table_keys</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">table_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">table_keys</span> <span class="o">=</span> <span class="n">table_keys</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">table_name</span> <span class="o">=</span> <span class="n">table_name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">region_name</span> <span class="o">=</span> <span class="n">region_name</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">AwsDynamoDBHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_resource_type</span><span class="p">(</span><span class="s1">'dynamodb'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">region_name</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span>
+
+<div class="viewcode-block" id="AwsDynamoDBHook.write_batch_data"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_dynamodb_hook.AwsDynamoDBHook.write_batch_data">[docs]</a> <span class="k">def</span> <span class="nf">write_batch_data</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">items</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Write batch items to dynamodb table with provisioned throughout capacity.</span>
+<span class="sd"> """</span>
+
+ <span class="n">dynamodb_conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">table</span> <span class="o">=</span> <span class="n">dynamodb_conn</span><span class="o">.</span><span class="n">Table</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">table_name</span><span class="p">)</span>
+
+ <span class="k">with</span> <span class="n">table</span><span class="o">.</span><span class="n">batch_writer</span><span class="p">(</span><span class="n">overwrite_by_pkeys</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_keys</span><span class="p">)</span> <span class="k">as</span> <span class="n">batch</span><span class="p">:</span>
+ <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">items</span><span class="p">:</span>
+ <span class="n">batch</span><span class="o">.</span><span class="n">put_item</span><span class="p">(</span><span class="n">Item</span><span class="o">=</span><span class="n">item</span><span class="p">)</span>
+ <span class="k">return</span> <span class="kc">True</span>
+ <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">general_error</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'Failed to insert items in dynamodb, error: </span><span class="si">{error}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">error</span><span class="o">=</span><span class="nb">str</span><span class="p">(</span><span class="n">general_error</span><span class="p">)</span>
+ <span class="p">)</span>
+ <span class="p">)</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/aws_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/aws_hook.html b/_modules/airflow/contrib/hooks/aws_hook.html
new file mode 100644
index 0000000..7d9a60f
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/aws_hook.html
@@ -0,0 +1,410 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.aws_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.aws_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.aws_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+
+<span class="kn">import</span> <span class="nn">boto3</span>
+<span class="kn">import</span> <span class="nn">configparser</span>
+<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+
+
+<span class="k">def</span> <span class="nf">_parse_s3_config</span><span class="p">(</span><span class="n">config_file_name</span><span class="p">,</span> <span class="n">config_format</span><span class="o">=</span><span class="s1">'boto'</span><span class="p">,</span> <span class="n">profile</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Parses a config file for s3 credentials. Can currently</span>
+<span class="sd"> parse boto, s3cmd.conf and AWS SDK config formats</span>
+
+<span class="sd"> :param config_file_name: path to the config file</span>
+<span class="sd"> :type config_file_name: str</span>
+<span class="sd"> :param config_format: config type. One of "boto", "s3cmd" or "aws".</span>
+<span class="sd"> Defaults to "boto"</span>
+<span class="sd"> :type config_format: str</span>
+<span class="sd"> :param profile: profile name in AWS type config file</span>
+<span class="sd"> :type profile: str</span>
+<span class="sd"> """</span>
+ <span class="n">config</span> <span class="o">=</span> <span class="n">configparser</span><span class="o">.</span><span class="n">ConfigParser</span><span class="p">()</span>
+ <span class="k">if</span> <span class="n">config</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="n">config_file_name</span><span class="p">):</span> <span class="c1"># pragma: no cover</span>
+ <span class="n">sections</span> <span class="o">=</span> <span class="n">config</span><span class="o">.</span><span class="n">sections</span><span class="p">()</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Couldn't read </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">config_file_name</span><span class="p">))</span>
+ <span class="c1"># Setting option names depending on file format</span>
+ <span class="k">if</span> <span class="n">config_format</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">config_format</span> <span class="o">=</span> <span class="s1">'boto'</span>
+ <span class="n">conf_format</span> <span class="o">=</span> <span class="n">config_format</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
+ <span class="k">if</span> <span class="n">conf_format</span> <span class="o">==</span> <span class="s1">'boto'</span><span class="p">:</span> <span class="c1"># pragma: no cover</span>
+ <span class="k">if</span> <span class="n">profile</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="s1">'profile '</span> <span class="o">+</span> <span class="n">profile</span> <span class="ow">in</span> <span class="n">sections</span><span class="p">:</span>
+ <span class="n">cred_section</span> <span class="o">=</span> <span class="s1">'profile '</span> <span class="o">+</span> <span class="n">profile</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">cred_section</span> <span class="o">=</span> <span class="s1">'Credentials'</span>
+ <span class="k">elif</span> <span class="n">conf_format</span> <span class="o">==</span> <span class="s1">'aws'</span> <span class="ow">and</span> <span class="n">profile</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">cred_section</span> <span class="o">=</span> <span class="n">profile</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">cred_section</span> <span class="o">=</span> <span class="s1">'default'</span>
+ <span class="c1"># Option names</span>
+ <span class="k">if</span> <span class="n">conf_format</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'boto'</span><span class="p">,</span> <span class="s1">'aws'</span><span class="p">):</span> <span class="c1"># pragma: no cover</span>
+ <span class="n">key_id_option</span> <span class="o">=</span> <span class="s1">'aws_access_key_id'</span>
+ <span class="n">secret_key_option</span> <span class="o">=</span> <span class="s1">'aws_secret_access_key'</span>
+ <span class="c1"># security_token_option = 'aws_security_token'</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">key_id_option</span> <span class="o">=</span> <span class="s1">'access_key'</span>
+ <span class="n">secret_key_option</span> <span class="o">=</span> <span class="s1">'secret_key'</span>
+ <span class="c1"># Actual Parsing</span>
+ <span class="k">if</span> <span class="n">cred_section</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">sections</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"This config file format is not recognized"</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">access_key</span> <span class="o">=</span> <span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">cred_section</span><span class="p">,</span> <span class="n">key_id_option</span><span class="p">)</span>
+ <span class="n">secret_key</span> <span class="o">=</span> <span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">cred_section</span><span class="p">,</span> <span class="n">secret_key_option</span><span class="p">)</span>
+ <span class="k">except</span><span class="p">:</span>
+ <span class="n">logging</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Option Error in parsing s3 config file"</span><span class="p">)</span>
+ <span class="k">raise</span>
+ <span class="k">return</span> <span class="n">access_key</span><span class="p">,</span> <span class="n">secret_key</span>
+
+
+<div class="viewcode-block" id="AwsHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_hook.AwsHook">[docs]</a><span class="k">class</span> <span class="nc">AwsHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with AWS.</span>
+<span class="sd"> This class is a thin wrapper around the boto3 python library.</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">aws_conn_id</span><span class="o">=</span><span class="s1">'aws_default'</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span> <span class="o">=</span> <span class="n">aws_conn_id</span>
+
+ <span class="k">def</span> <span class="nf">_get_credentials</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">region_name</span><span class="p">):</span>
+ <span class="n">aws_access_key_id</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="n">aws_secret_access_key</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="n">aws_session_token</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="n">endpoint_url</span> <span class="o">=</span> <span class="kc">None</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">:</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">connection_object</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">login</span><span class="p">:</span>
+ <span class="n">aws_access_key_id</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">login</span>
+ <span class="n">aws_secret_access_key</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">password</span>
+
+ <span class="k">elif</span> <span class="s1">'aws_secret_access_key'</span> <span class="ow">in</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">:</span>
+ <span class="n">aws_access_key_id</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s1">'aws_access_key_id'</span><span class="p">]</span>
+ <span class="n">aws_secret_access_key</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s1">'aws_secret_access_key'</span><span class="p">]</span>
+
+ <span class="k">elif</span> <span class="s1">'s3_config_file'</span> <span class="ow">in</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">:</span>
+ <span class="n">aws_access_key_id</span><span class="p">,</span> <span class="n">aws_secret_access_key</span> <span class="o">=</span> \
+ <span class="n">_parse_s3_config</span><span class="p">(</span><span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s1">'s3_config_file'</span><span class="p">],</span>
+ <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'s3_config_format'</span><span class="p">))</span>
+
+ <span class="k">if</span> <span class="n">region_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">region_name</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'region_name'</span><span class="p">)</span>
+
+ <span class="n">role_arn</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'role_arn'</span><span class="p">)</span>
+ <span class="n">aws_account_id</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'aws_account_id'</span><span class="p">)</span>
+ <span class="n">aws_iam_role</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'aws_iam_role'</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">role_arn</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">aws_account_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> \
+ <span class="n">aws_iam_role</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+
+ <span class="n">role_arn</span> <span class="o">=</span> <span class="s2">"arn:aws:iam::"</span> <span class="o">+</span> <span class="n">aws_account_id</span> <span class="o">+</span> <span class="s2">":role/"</span> <span class="o">+</span> <span class="n">aws_iam_role</span>
+
+ <span class="k">if</span> <span class="n">role_arn</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">sts_session</span> <span class="o">=</span> <span class="n">boto3</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">Session</span><span class="p">(</span>
+ <span class="n">aws_access_key_id</span><span class="o">=</span><span class="n">aws_access_key_id</span><span class="p">,</span>
+ <span class="n">aws_secret_access_key</span><span class="o">=</span><span class="n">aws_secret_access_key</span><span class="p">,</span>
+ <span class="n">region_name</span><span class="o">=</span><span class="n">region_name</span><span class="p">)</span>
+
+ <span class="n">sts_client</span> <span class="o">=</span> <span class="n">sts_session</span><span class="o">.</span><span class="n">client</span><span class="p">(</span><span class="s1">'sts'</span><span class="p">)</span>
+ <span class="n">sts_response</span> <span class="o">=</span> <span class="n">sts_client</span><span class="o">.</span><span class="n">assume_role</span><span class="p">(</span>
+ <span class="n">RoleArn</span><span class="o">=</span><span class="n">role_arn</span><span class="p">,</span>
+ <span class="n">RoleSessionName</span><span class="o">=</span><span class="s1">'Airflow_'</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span>
+ <span class="n">aws_access_key_id</span> <span class="o">=</span> <span class="n">sts_response</span><span class="p">[</span><span class="s1">'Credentials'</span><span class="p">][</span><span class="s1">'AccessKeyId'</span><span class="p">]</span>
+ <span class="n">aws_secret_access_key</span> <span class="o">=</span> <span class="n">sts_response</span><span class="p">[</span><span class="s1">'Credentials'</span><span class="p">][</span><span class="s1">'SecretAccessKey'</span><span class="p">]</span>
+ <span class="n">aws_session_token</span> <span class="o">=</span> <span class="n">sts_response</span><span class="p">[</span><span class="s1">'Credentials'</span><span class="p">][</span><span class="s1">'SessionToken'</span><span class="p">]</span>
+
+ <span class="n">endpoint_url</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'host'</span><span class="p">)</span>
+
+ <span class="k">except</span> <span class="n">AirflowException</span><span class="p">:</span>
+ <span class="c1"># No connection found: fallback on boto3 credential strategy</span>
+ <span class="c1"># http://boto3.readthedocs.io/en/latest/guide/configuration.html</span>
+ <span class="k">pass</span>
+
+ <span class="k">return</span> <span class="n">boto3</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">Session</span><span class="p">(</span>
+ <span class="n">aws_access_key_id</span><span class="o">=</span><span class="n">aws_access_key_id</span><span class="p">,</span>
+ <span class="n">aws_secret_access_key</span><span class="o">=</span><span class="n">aws_secret_access_key</span><span class="p">,</span>
+ <span class="n">aws_session_token</span><span class="o">=</span><span class="n">aws_session_token</span><span class="p">,</span>
+ <span class="n">region_name</span><span class="o">=</span><span class="n">region_name</span><span class="p">),</span> <span class="n">endpoint_url</span>
+
+ <span class="k">def</span> <span class="nf">get_client_type</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">client_type</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="n">session</span><span class="p">,</span> <span class="n">endpoint_url</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_credentials</span><span class="p">(</span><span class="n">region_name</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">client</span><span class="p">(</span><span class="n">client_type</span><span class="p">,</span> <span class="n">endpoint_url</span><span class="o">=</span><span class="n">endpoint_url</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">get_resource_type</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resource_type</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="n">session</span><span class="p">,</span> <span class="n">endpoint_url</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_credentials</span><span class="p">(</span><span class="n">region_name</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">resource</span><span class="p">(</span><span class="n">resource_type</span><span class="p">,</span> <span class="n">endpoint_url</span><span class="o">=</span><span class="n">endpoint_url</span><span class="p">)</span>
+
+<div class="viewcode-block" id="AwsHook.get_session"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_hook.AwsHook.get_session">[docs]</a> <span class="k">def</span> <span class="nf">get_session</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""Get the underlying boto3.session."""</span>
+ <span class="n">session</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_credentials</span><span class="p">(</span><span class="n">region_name</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">session</span></div>
+
+<div class="viewcode-block" id="AwsHook.get_credentials"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_hook.AwsHook.get_credentials">[docs]</a> <span class="k">def</span> <span class="nf">get_credentials</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""Get the underlying `botocore.Credentials` object.</span>
+
+<span class="sd"> This contains the attributes: access_key, secret_key and token.</span>
+<span class="sd"> """</span>
+ <span class="n">session</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_credentials</span><span class="p">(</span><span class="n">region_name</span><span class="p">)</span>
+ <span class="c1"># Credentials are refreshable, so accessing your access key / secret key</span>
+ <span class="c1"># separately can lead to a race condition.</span>
+ <span class="c1"># See https://stackoverflow.com/a/36291428/8283373</span>
+ <span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">get_credentials</span><span class="p">()</span><span class="o">.</span><span class="n">get_frozen_credentials</span><span class="p">()</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/aws_lambda_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/aws_lambda_hook.html b/_modules/airflow/contrib/hooks/aws_lambda_hook.html
new file mode 100644
index 0000000..5953a6b
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/aws_lambda_hook.html
@@ -0,0 +1,302 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.aws_lambda_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.aws_lambda_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.aws_lambda_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.aws_hook</span> <span class="k">import</span> <span class="n">AwsHook</span>
+
+
+<div class="viewcode-block" id="AwsLambdaHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_lambda_hook.AwsLambdaHook">[docs]</a><span class="k">class</span> <span class="nc">AwsLambdaHook</span><span class="p">(</span><span class="n">AwsHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with AWS Lambda</span>
+
+<span class="sd"> :param function_name: AWS Lambda Function Name</span>
+<span class="sd"> :type function_name: str</span>
+<span class="sd"> :param region_name: AWS Region Name (example: us-west-2)</span>
+<span class="sd"> :type region_name: str</span>
+<span class="sd"> :param log_type: Tail Invocation Request</span>
+<span class="sd"> :type log_type: str</span>
+<span class="sd"> :param qualifier: AWS Lambda Function Version or Alias Name</span>
+<span class="sd"> :type qualifier: str</span>
+<span class="sd"> :param invocation_type: AWS Lambda Invocation Type (RequestResponse, Event etc)</span>
+<span class="sd"> :type invocation_type: str</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">function_name</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">log_type</span><span class="o">=</span><span class="s1">'None'</span><span class="p">,</span> <span class="n">qualifier</span><span class="o">=</span><span class="s1">'$LATEST'</span><span class="p">,</span>
+ <span class="n">invocation_type</span><span class="o">=</span><span class="s1">'RequestResponse'</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">function_name</span> <span class="o">=</span> <span class="n">function_name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">region_name</span> <span class="o">=</span> <span class="n">region_name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log_type</span> <span class="o">=</span> <span class="n">log_type</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">invocation_type</span> <span class="o">=</span> <span class="n">invocation_type</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">qualifier</span> <span class="o">=</span> <span class="n">qualifier</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">AwsLambdaHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_client_type</span><span class="p">(</span><span class="s1">'lambda'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">region_name</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span>
+
+<div class="viewcode-block" id="AwsLambdaHook.invoke_lambda"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_lambda_hook.AwsLambdaHook.invoke_lambda">[docs]</a> <span class="k">def</span> <span class="nf">invoke_lambda</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">payload</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Invoke Lambda Function</span>
+<span class="sd"> """</span>
+
+ <span class="n">awslambda_conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+ <span class="n">response</span> <span class="o">=</span> <span class="n">awslambda_conn</span><span class="o">.</span><span class="n">invoke</span><span class="p">(</span>
+ <span class="n">FunctionName</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">function_name</span><span class="p">,</span>
+ <span class="n">InvocationType</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">invocation_type</span><span class="p">,</span>
+ <span class="n">LogType</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">log_type</span><span class="p">,</span>
+ <span class="n">Payload</span><span class="o">=</span><span class="n">payload</span><span class="p">,</span>
+ <span class="n">Qualifier</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">qualifier</span>
+ <span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">response</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[38/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/sqoop_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/sqoop_hook.html b/_modules/airflow/contrib/hooks/sqoop_hook.html
new file mode 100644
index 0000000..4563030
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/sqoop_hook.html
@@ -0,0 +1,580 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.sqoop_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.sqoop_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.sqoop_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+
+<span class="sd">"""</span>
+<span class="sd">This module contains a sqoop 1.x hook</span>
+<span class="sd">"""</span>
+<span class="kn">import</span> <span class="nn">subprocess</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+<span class="kn">from</span> <span class="nn">copy</span> <span class="k">import</span> <span class="n">deepcopy</span>
+
+
+<div class="viewcode-block" id="SqoopHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sqoop_hook.SqoopHook">[docs]</a><span class="k">class</span> <span class="nc">SqoopHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> This hook is a wrapper around the sqoop 1 binary. To be able to use the hook</span>
+<span class="sd"> it is required that "sqoop" is in the PATH.</span>
+
+<span class="sd"> Additional arguments that can be passed via the 'extra' JSON field of the</span>
+<span class="sd"> sqoop connection:</span>
+<span class="sd"> * job_tracker: Job tracker local|jobtracker:port.</span>
+<span class="sd"> * namenode: Namenode.</span>
+<span class="sd"> * lib_jars: Comma separated jar files to include in the classpath.</span>
+<span class="sd"> * files: Comma separated files to be copied to the map reduce cluster.</span>
+<span class="sd"> * archives: Comma separated archives to be unarchived on the compute</span>
+<span class="sd"> machines.</span>
+<span class="sd"> * password_file: Path to file containing the password.</span>
+
+<span class="sd"> :param conn_id: Reference to the sqoop connection.</span>
+<span class="sd"> :type conn_id: str</span>
+<span class="sd"> :param verbose: Set sqoop to verbose.</span>
+<span class="sd"> :type verbose: bool</span>
+<span class="sd"> :param num_mappers: Number of map tasks to import in parallel.</span>
+<span class="sd"> :type num_mappers: int</span>
+<span class="sd"> :param properties: Properties to set via the -D argument</span>
+<span class="sd"> :type properties: dict</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn_id</span><span class="o">=</span><span class="s1">'sqoop_default'</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">num_mappers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">hcatalog_database</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">hcatalog_table</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">properties</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="c1"># No mutable types in the default parameters</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">conn_id</span><span class="p">)</span>
+ <span class="n">connection_parameters</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job_tracker</span> <span class="o">=</span> <span class="n">connection_parameters</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'job_tracker'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">namenode</span> <span class="o">=</span> <span class="n">connection_parameters</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'namenode'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">libjars</span> <span class="o">=</span> <span class="n">connection_parameters</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'libjars'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">files</span> <span class="o">=</span> <span class="n">connection_parameters</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'files'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">archives</span> <span class="o">=</span> <span class="n">connection_parameters</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'archives'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">password_file</span> <span class="o">=</span> <span class="n">connection_parameters</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'password_file'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hcatalog_database</span> <span class="o">=</span> <span class="n">hcatalog_database</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hcatalog_table</span> <span class="o">=</span> <span class="n">hcatalog_table</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">num_mappers</span> <span class="o">=</span> <span class="n">num_mappers</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">properties</span> <span class="o">=</span> <span class="n">properties</span> <span class="ow">or</span> <span class="p">{}</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Using connection to: </span><span class="si">{}</span><span class="s2">:</span><span class="si">{}</span><span class="s2">/</span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">schema</span><span class="p">))</span>
+
+ <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span>
+
+ <span class="k">def</span> <span class="nf">cmd_mask_password</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cmd_orig</span><span class="p">):</span>
+ <span class="n">cmd</span> <span class="o">=</span> <span class="n">deepcopy</span><span class="p">(</span><span class="n">cmd_orig</span><span class="p">)</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">password_index</span> <span class="o">=</span> <span class="n">cmd</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="s1">'--password'</span><span class="p">)</span>
+ <span class="n">cmd</span><span class="p">[</span><span class="n">password_index</span> <span class="o">+</span> <span class="mi">1</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'MASKED'</span>
+ <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"No password in sqoop cmd"</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">cmd</span>
+
+<div class="viewcode-block" id="SqoopHook.Popen"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sqoop_hook.SqoopHook.Popen">[docs]</a> <span class="k">def</span> <span class="nf">Popen</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cmd</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Remote Popen</span>
+
+<span class="sd"> :param cmd: command to remotely execute</span>
+<span class="sd"> :param kwargs: extra arguments to Popen (see subprocess.Popen)</span>
+<span class="sd"> :return: handle to subprocess</span>
+<span class="sd"> """</span>
+ <span class="n">masked_cmd</span> <span class="o">=</span> <span class="s1">' '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cmd_mask_password</span><span class="p">(</span><span class="n">cmd</span><span class="p">))</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Executing command: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">masked_cmd</span><span class="p">))</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sp</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">Popen</span><span class="p">(</span><span class="n">cmd</span><span class="p">,</span>
+ <span class="n">stdout</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span>
+ <span class="n">stderr</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">STDOUT</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="nb">iter</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sp</span><span class="o">.</span><span class="n">stdout</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">line</span><span class="o">.</span><span class="n">strip</span><span class="p">())</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">sp</span><span class="o">.</span><span class="n">wait</span><span class="p">()</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Command exited with return code </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sp</span><span class="o">.</span><span class="n">returncode</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">sp</span><span class="o">.</span><span class="n">returncode</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Sqoop command failed: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">masked_cmd</span><span class="p">))</span></div>
+
+ <span class="k">def</span> <span class="nf">_prepare_command</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">export</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+ <span class="n">sqoop_cmd_type</span> <span class="o">=</span> <span class="s2">"export"</span> <span class="k">if</span> <span class="n">export</span> <span class="k">else</span> <span class="s2">"import"</span>
+ <span class="n">connection_cmd</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"sqoop"</span><span class="p">,</span> <span class="n">sqoop_cmd_type</span><span class="p">]</span>
+
+ <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">properties</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"-D"</span><span class="p">,</span> <span class="s2">"</span><span class="si">{}</span><span class="s2">=</span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">)]</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">namenode</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"-fs"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">namenode</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_tracker</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"-jt"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_tracker</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">libjars</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"-libjars"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">libjars</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">files</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"-files"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">files</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">archives</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"-archives"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">archives</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--username"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--password"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">password_file</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--password-file"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">password_file</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--verbose"</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_mappers</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--num-mappers"</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">num_mappers</span><span class="p">)]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">hcatalog_database</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--hcatalog-database"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">hcatalog_database</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">hcatalog_table</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--hcatalog-table"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">hcatalog_table</span><span class="p">]</span>
+
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--connect"</span><span class="p">,</span> <span class="s2">"</span><span class="si">{}</span><span class="s2">:</span><span class="si">{}</span><span class="s2">/</span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">schema</span>
+ <span class="p">)]</span>
+
+ <span class="k">return</span> <span class="n">connection_cmd</span>
+
+ <span class="nd">@staticmethod</span>
+ <span class="k">def</span> <span class="nf">_get_export_format_argument</span><span class="p">(</span><span class="n">file_type</span><span class="o">=</span><span class="s1">'text'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">file_type</span> <span class="o">==</span> <span class="s2">"avro"</span><span class="p">:</span>
+ <span class="k">return</span> <span class="p">[</span><span class="s2">"--as-avrodatafile"</span><span class="p">]</span>
+ <span class="k">elif</span> <span class="n">file_type</span> <span class="o">==</span> <span class="s2">"sequence"</span><span class="p">:</span>
+ <span class="k">return</span> <span class="p">[</span><span class="s2">"--as-sequencefile"</span><span class="p">]</span>
+ <span class="k">elif</span> <span class="n">file_type</span> <span class="o">==</span> <span class="s2">"parquet"</span><span class="p">:</span>
+ <span class="k">return</span> <span class="p">[</span><span class="s2">"--as-parquetfile"</span><span class="p">]</span>
+ <span class="k">elif</span> <span class="n">file_type</span> <span class="o">==</span> <span class="s2">"text"</span><span class="p">:</span>
+ <span class="k">return</span> <span class="p">[</span><span class="s2">"--as-textfile"</span><span class="p">]</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Argument file_type should be 'avro', "</span>
+ <span class="s2">"'sequence', 'parquet' or 'text'."</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_import_cmd</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">target_dir</span><span class="p">,</span> <span class="n">append</span><span class="p">,</span> <span class="n">file_type</span><span class="p">,</span> <span class="n">split_by</span><span class="p">,</span> <span class="n">direct</span><span class="p">,</span>
+ <span class="n">driver</span><span class="p">,</span> <span class="n">extra_import_options</span><span class="p">):</span>
+
+ <span class="n">cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_command</span><span class="p">(</span><span class="n">export</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">target_dir</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--target-dir"</span><span class="p">,</span> <span class="n">target_dir</span><span class="p">]</span>
+
+ <span class="k">if</span> <span class="n">append</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--append"</span><span class="p">]</span>
+
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_export_format_argument</span><span class="p">(</span><span class="n">file_type</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">split_by</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--split-by"</span><span class="p">,</span> <span class="n">split_by</span><span class="p">]</span>
+
+ <span class="k">if</span> <span class="n">direct</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--direct"</span><span class="p">]</span>
+
+ <span class="k">if</span> <span class="n">driver</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--driver"</span><span class="p">,</span> <span class="n">driver</span><span class="p">]</span>
+
+ <span class="k">if</span> <span class="n">extra_import_options</span><span class="p">:</span>
+ <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">extra_import_options</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'--</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="p">)]</span>
+ <span class="k">if</span> <span class="n">value</span><span class="p">:</span> <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="n">value</span><span class="p">]</span>
+
+ <span class="k">return</span> <span class="n">cmd</span>
+
+<div class="viewcode-block" id="SqoopHook.import_table"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sqoop_hook.SqoopHook.import_table">[docs]</a> <span class="k">def</span> <span class="nf">import_table</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">target_dir</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">append</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">file_type</span><span class="o">=</span><span class="s2">"text"</span><span class="p">,</span>
+ <span class="n">columns</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">split_by</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">where</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">direct</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">driver</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">extra_import_options</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Imports table from remote location to target dir. Arguments are</span>
+<span class="sd"> copies of direct sqoop command line arguments</span>
+<span class="sd"> :param table: Table to read</span>
+<span class="sd"> :param target_dir: HDFS destination dir</span>
+<span class="sd"> :param append: Append data to an existing dataset in HDFS</span>
+<span class="sd"> :param file_type: "avro", "sequence", "text" or "parquet".</span>
+<span class="sd"> Imports data to into the specified format. Defaults to text.</span>
+<span class="sd"> :param columns: <col,col,col…> Columns to import from table</span>
+<span class="sd"> :param split_by: Column of the table used to split work units</span>
+<span class="sd"> :param where: WHERE clause to use during import</span>
+<span class="sd"> :param direct: Use direct connector if exists for the database</span>
+<span class="sd"> :param driver: Manually specify JDBC driver class to use</span>
+<span class="sd"> :param extra_import_options: Extra import options to pass as dict.</span>
+<span class="sd"> If a key doesn't have a value, just pass an empty string to it.</span>
+<span class="sd"> Don't include prefix of -- for sqoop options.</span>
+<span class="sd"> """</span>
+ <span class="n">cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_import_cmd</span><span class="p">(</span><span class="n">target_dir</span><span class="p">,</span> <span class="n">append</span><span class="p">,</span> <span class="n">file_type</span><span class="p">,</span> <span class="n">split_by</span><span class="p">,</span> <span class="n">direct</span><span class="p">,</span>
+ <span class="n">driver</span><span class="p">,</span> <span class="n">extra_import_options</span><span class="p">)</span>
+
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--table"</span><span class="p">,</span> <span class="n">table</span><span class="p">]</span>
+
+ <span class="k">if</span> <span class="n">columns</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--columns"</span><span class="p">,</span> <span class="n">columns</span><span class="p">]</span>
+ <span class="k">if</span> <span class="n">where</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--where"</span><span class="p">,</span> <span class="n">where</span><span class="p">]</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">Popen</span><span class="p">(</span><span class="n">cmd</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="SqoopHook.import_query"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sqoop_hook.SqoopHook.import_query">[docs]</a> <span class="k">def</span> <span class="nf">import_query</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query</span><span class="p">,</span> <span class="n">target_dir</span><span class="p">,</span> <span class="n">append</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">file_type</span><span class="o">=</span><span class="s2">"text"</span><span class="p">,</span>
+ <span class="n">split_by</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">direct</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">driver</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">extra_import_options</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Imports a specific query from the rdbms to hdfs</span>
+<span class="sd"> :param query: Free format query to run</span>
+<span class="sd"> :param target_dir: HDFS destination dir</span>
+<span class="sd"> :param append: Append data to an existing dataset in HDFS</span>
+<span class="sd"> :param file_type: "avro", "sequence", "text" or "parquet"</span>
+<span class="sd"> Imports data to hdfs into the specified format. Defaults to text.</span>
+<span class="sd"> :param split_by: Column of the table used to split work units</span>
+<span class="sd"> :param direct: Use direct import fast path</span>
+<span class="sd"> :param driver: Manually specify JDBC driver class to use</span>
+<span class="sd"> :param extra_import_options: Extra import options to pass as dict.</span>
+<span class="sd"> If a key doesn't have a value, just pass an empty string to it.</span>
+<span class="sd"> Don't include prefix of -- for sqoop options.</span>
+<span class="sd"> """</span>
+ <span class="n">cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_import_cmd</span><span class="p">(</span><span class="n">target_dir</span><span class="p">,</span> <span class="n">append</span><span class="p">,</span> <span class="n">file_type</span><span class="p">,</span> <span class="n">split_by</span><span class="p">,</span> <span class="n">direct</span><span class="p">,</span>
+ <span class="n">driver</span><span class="p">,</span> <span class="n">extra_import_options</span><span class="p">)</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--query"</span><span class="p">,</span> <span class="n">query</span><span class="p">]</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">Popen</span><span class="p">(</span><span class="n">cmd</span><span class="p">)</span></div>
+
+ <span class="k">def</span> <span class="nf">_export_cmd</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">export_dir</span><span class="p">,</span> <span class="n">input_null_string</span><span class="p">,</span>
+ <span class="n">input_null_non_string</span><span class="p">,</span> <span class="n">staging_table</span><span class="p">,</span> <span class="n">clear_staging_table</span><span class="p">,</span>
+ <span class="n">enclosed_by</span><span class="p">,</span> <span class="n">escaped_by</span><span class="p">,</span> <span class="n">input_fields_terminated_by</span><span class="p">,</span>
+ <span class="n">input_lines_terminated_by</span><span class="p">,</span> <span class="n">input_optionally_enclosed_by</span><span class="p">,</span>
+ <span class="n">batch</span><span class="p">,</span> <span class="n">relaxed_isolation</span><span class="p">,</span> <span class="n">extra_export_options</span><span class="p">):</span>
+
+ <span class="n">cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_command</span><span class="p">(</span><span class="n">export</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">input_null_string</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--input-null-string"</span><span class="p">,</span> <span class="n">input_null_string</span><span class="p">]</span>
+
+ <span class="k">if</span> <span class="n">input_null_non_string</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--input-null-non-string"</span><span class="p">,</span> <span class="n">input_null_non_string</span><span class="p">]</span>
+
+ <span class="k">if</span> <span class="n">staging_table</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--staging-table"</span><span class="p">,</span> <span class="n">staging_table</span><span class="p">]</span>
+
+ <span class="k">if</span> <span class="n">clear_staging_table</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--clear-staging-table"</span><span class="p">]</span>
+
+ <span class="k">if</span> <span class="n">enclosed_by</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--enclosed-by"</span><span class="p">,</span> <span class="n">enclosed_by</span><span class="p">]</span>
+
+ <span class="k">if</span> <span class="n">escaped_by</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--escaped-by"</span><span class="p">,</span> <span class="n">escaped_by</span><span class="p">]</span>
+
+ <span class="k">if</span> <span class="n">input_fields_terminated_by</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--input-fields-terminated-by"</span><span class="p">,</span> <span class="n">input_fields_terminated_by</span><span class="p">]</span>
+
+ <span class="k">if</span> <span class="n">input_lines_terminated_by</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--input-lines-terminated-by"</span><span class="p">,</span> <span class="n">input_lines_terminated_by</span><span class="p">]</span>
+
+ <span class="k">if</span> <span class="n">input_optionally_enclosed_by</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--input-optionally-enclosed-by"</span><span class="p">,</span>
+ <span class="n">input_optionally_enclosed_by</span><span class="p">]</span>
+
+ <span class="k">if</span> <span class="n">batch</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--batch"</span><span class="p">]</span>
+
+ <span class="k">if</span> <span class="n">relaxed_isolation</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--relaxed-isolation"</span><span class="p">]</span>
+
+ <span class="k">if</span> <span class="n">export_dir</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--export-dir"</span><span class="p">,</span> <span class="n">export_dir</span><span class="p">]</span>
+
+ <span class="k">if</span> <span class="n">extra_export_options</span><span class="p">:</span>
+ <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">extra_export_options</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'--</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="p">)]</span>
+ <span class="k">if</span> <span class="n">value</span><span class="p">:</span> <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="n">value</span><span class="p">]</span>
+
+ <span class="c1"># The required option</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--table"</span><span class="p">,</span> <span class="n">table</span><span class="p">]</span>
+
+ <span class="k">return</span> <span class="n">cmd</span>
+
+<div class="viewcode-block" id="SqoopHook.export_table"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sqoop_hook.SqoopHook.export_table">[docs]</a> <span class="k">def</span> <span class="nf">export_table</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">export_dir</span><span class="p">,</span> <span class="n">input_null_string</span><span class="p">,</span>
+ <span class="n">input_null_non_string</span><span class="p">,</span> <span class="n">staging_table</span><span class="p">,</span>
+ <span class="n">clear_staging_table</span><span class="p">,</span> <span class="n">enclosed_by</span><span class="p">,</span>
+ <span class="n">escaped_by</span><span class="p">,</span> <span class="n">input_fields_terminated_by</span><span class="p">,</span>
+ <span class="n">input_lines_terminated_by</span><span class="p">,</span>
+ <span class="n">input_optionally_enclosed_by</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span>
+ <span class="n">relaxed_isolation</span><span class="p">,</span> <span class="n">extra_export_options</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Exports Hive table to remote location. Arguments are copies of direct</span>
+<span class="sd"> sqoop command line Arguments</span>
+<span class="sd"> :param table: Table remote destination</span>
+<span class="sd"> :param export_dir: Hive table to export</span>
+<span class="sd"> :param input_null_string: The string to be interpreted as null for</span>
+<span class="sd"> string columns</span>
+<span class="sd"> :param input_null_non_string: The string to be interpreted as null</span>
+<span class="sd"> for non-string columns</span>
+<span class="sd"> :param staging_table: The table in which data will be staged before</span>
+<span class="sd"> being inserted into the destination table</span>
+<span class="sd"> :param clear_staging_table: Indicate that any data present in the</span>
+<span class="sd"> staging table can be deleted</span>
+<span class="sd"> :param enclosed_by: Sets a required field enclosing character</span>
+<span class="sd"> :param escaped_by: Sets the escape character</span>
+<span class="sd"> :param input_fields_terminated_by: Sets the field separator character</span>
+<span class="sd"> :param input_lines_terminated_by: Sets the end-of-line character</span>
+<span class="sd"> :param input_optionally_enclosed_by: Sets a field enclosing character</span>
+<span class="sd"> :param batch: Use batch mode for underlying statement execution</span>
+<span class="sd"> :param relaxed_isolation: Transaction isolation to read uncommitted</span>
+<span class="sd"> for the mappers</span>
+<span class="sd"> :param extra_export_options: Extra export options to pass as dict.</span>
+<span class="sd"> If a key doesn't have a value, just pass an empty string to it.</span>
+<span class="sd"> Don't include prefix of -- for sqoop options.</span>
+<span class="sd"> """</span>
+ <span class="n">cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_export_cmd</span><span class="p">(</span><span class="n">table</span><span class="p">,</span> <span class="n">export_dir</span><span class="p">,</span> <span class="n">input_null_string</span><span class="p">,</span>
+ <span class="n">input_null_non_string</span><span class="p">,</span> <span class="n">staging_table</span><span class="p">,</span>
+ <span class="n">clear_staging_table</span><span class="p">,</span> <span class="n">enclosed_by</span><span class="p">,</span> <span class="n">escaped_by</span><span class="p">,</span>
+ <span class="n">input_fields_terminated_by</span><span class="p">,</span>
+ <span class="n">input_lines_terminated_by</span><span class="p">,</span>
+ <span class="n">input_optionally_enclosed_by</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span>
+ <span class="n">relaxed_isolation</span><span class="p">,</span> <span class="n">extra_export_options</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">Popen</span><span class="p">(</span><span class="n">cmd</span><span class="p">)</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[04/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/sensors/hdfs_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/sensors/hdfs_sensor.html b/_modules/airflow/sensors/hdfs_sensor.html
new file mode 100644
index 0000000..7747a54
--- /dev/null
+++ b/_modules/airflow/sensors/hdfs_sensor.html
@@ -0,0 +1,352 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.sensors.hdfs_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.sensors.hdfs_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.sensors.hdfs_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">re</span>
+<span class="kn">import</span> <span class="nn">sys</span>
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
+
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">settings</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.hdfs_hook</span> <span class="k">import</span> <span class="n">HDFSHook</span>
+<span class="kn">from</span> <span class="nn">airflow.sensors.base_sensor_operator</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+
+<div class="viewcode-block" id="HdfsSensor"><a class="viewcode-back" href="../../../code.html#airflow.sensors.hdfs_sensor.HdfsSensor">[docs]</a><span class="k">class</span> <span class="nc">HdfsSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Waits for a file or folder to land in HDFS</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'filepath'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="n">settings</span><span class="o">.</span><span class="n">WEB_COLORS</span><span class="p">[</span><span class="s1">'LIGHTBLUE'</span><span class="p">]</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">filepath</span><span class="p">,</span>
+ <span class="n">hdfs_conn_id</span><span class="o">=</span><span class="s1">'hdfs_default'</span><span class="p">,</span>
+ <span class="n">ignored_ext</span><span class="o">=</span><span class="p">[</span><span class="s1">'_COPYING_'</span><span class="p">],</span>
+ <span class="n">ignore_copying</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+ <span class="n">file_size</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">hook</span><span class="o">=</span><span class="n">HDFSHook</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HdfsSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">filepath</span> <span class="o">=</span> <span class="n">filepath</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span> <span class="o">=</span> <span class="n">hdfs_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">file_size</span> <span class="o">=</span> <span class="n">file_size</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">ignored_ext</span> <span class="o">=</span> <span class="n">ignored_ext</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">ignore_copying</span> <span class="o">=</span> <span class="n">ignore_copying</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">hook</span>
+
+<div class="viewcode-block" id="HdfsSensor.filter_for_filesize"><a class="viewcode-back" href="../../../code.html#airflow.sensors.hdfs_sensor.HdfsSensor.filter_for_filesize">[docs]</a> <span class="nd">@staticmethod</span>
+ <span class="k">def</span> <span class="nf">filter_for_filesize</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">size</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Will test the filepath result and test if its size is at least self.filesize</span>
+
+<span class="sd"> :param result: a list of dicts returned by Snakebite ls</span>
+<span class="sd"> :param size: the file size in MB a file should be at least to trigger True</span>
+<span class="sd"> :return: (bool) depending on the matching criteria</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">size</span><span class="p">:</span>
+ <span class="n">log</span> <span class="o">=</span> <span class="n">LoggingMixin</span><span class="p">()</span><span class="o">.</span><span class="n">log</span>
+ <span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
+ <span class="s1">'Filtering for file size >= </span><span class="si">%s</span><span class="s1"> in files: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">size</span><span class="p">,</span> <span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="s1">'path'</span><span class="p">],</span> <span class="n">result</span><span class="p">)</span>
+ <span class="p">)</span>
+ <span class="n">size</span> <span class="o">*=</span> <span class="n">settings</span><span class="o">.</span><span class="n">MEGABYTE</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="p">[</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">result</span> <span class="k">if</span> <span class="n">x</span><span class="p">[</span><span class="s1">'length'</span><span class="p">]</span> <span class="o">>=</span> <span class="n">size</span><span class="p">]</span>
+ <span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'HdfsSensor.poke: after size filter result is </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">result</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">result</span></div>
+
+<div class="viewcode-block" id="HdfsSensor.filter_for_ignored_ext"><a class="viewcode-back" href="../../../code.html#airflow.sensors.hdfs_sensor.HdfsSensor.filter_for_ignored_ext">[docs]</a> <span class="nd">@staticmethod</span>
+ <span class="k">def</span> <span class="nf">filter_for_ignored_ext</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">ignored_ext</span><span class="p">,</span> <span class="n">ignore_copying</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Will filter if instructed to do so the result to remove matching criteria</span>
+
+<span class="sd"> :param result: (list) of dicts returned by Snakebite ls</span>
+<span class="sd"> :param ignored_ext: (list) of ignored extensions</span>
+<span class="sd"> :param ignore_copying: (bool) shall we ignore ?</span>
+<span class="sd"> :return: (list) of dicts which were not removed</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">ignore_copying</span><span class="p">:</span>
+ <span class="n">log</span> <span class="o">=</span> <span class="n">LoggingMixin</span><span class="p">()</span><span class="o">.</span><span class="n">log</span>
+ <span class="n">regex_builder</span> <span class="o">=</span> <span class="s2">"^.*\.(</span><span class="si">%s</span><span class="s2">$)$"</span> <span class="o">%</span> <span class="s1">'$|'</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">ignored_ext</span><span class="p">)</span>
+ <span class="n">ignored_extentions_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="n">regex_builder</span><span class="p">)</span>
+ <span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
+ <span class="s1">'Filtering result for ignored extensions: </span><span class="si">%s</span><span class="s1"> in files </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">ignored_extentions_regex</span><span class="o">.</span><span class="n">pattern</span><span class="p">,</span> <span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="s1">'path'</span><span class="p">],</span> <span class="n">result</span><span class="p">)</span>
+ <span class="p">)</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="p">[</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">result</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">ignored_extentions_regex</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="s1">'path'</span><span class="p">])]</span>
+ <span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'HdfsSensor.poke: after ext filter result is </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">result</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">result</span></div>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">sb</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Poking for file </span><span class="si">{self.filepath}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="c1"># IMOO it's not right here, as there no raise of any kind.</span>
+ <span class="c1"># if the filepath is let's say '/data/mydirectory',</span>
+ <span class="c1"># it's correct but if it is '/data/mydirectory/*',</span>
+ <span class="c1"># it's not correct as the directory exists and sb does not raise any error</span>
+ <span class="c1"># here is a quick fix</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="p">[</span><span class="n">f</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">sb</span><span class="o">.</span><span class="n">ls</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="p">],</span> <span class="n">include_toplevel</span><span class="o">=</span><span class="kc">False</span><span class="p">)]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'HdfsSensor.poke: result is </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">result</span><span class="p">)</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">filter_for_ignored_ext</span><span class="p">(</span>
+ <span class="n">result</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ignored_ext</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ignore_copying</span>
+ <span class="p">)</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">filter_for_filesize</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">file_size</span><span class="p">)</span>
+ <span class="k">return</span> <span class="nb">bool</span><span class="p">(</span><span class="n">result</span><span class="p">)</span>
+ <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
+ <span class="n">e</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">exc_info</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Caught an exception !: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
+ <span class="k">return</span> <span class="kc">False</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/sensors/hive_partition_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/sensors/hive_partition_sensor.html b/_modules/airflow/sensors/hive_partition_sensor.html
new file mode 100644
index 0000000..5549f84
--- /dev/null
+++ b/_modules/airflow/sensors/hive_partition_sensor.html
@@ -0,0 +1,310 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.sensors.hive_partition_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.sensors.hive_partition_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.sensors.hive_partition_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.sensors.base_sensor_operator</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="HivePartitionSensor"><a class="viewcode-back" href="../../../code.html#airflow.sensors.hive_partition_sensor.HivePartitionSensor">[docs]</a><span class="k">class</span> <span class="nc">HivePartitionSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Waits for a partition to show up in Hive.</span>
+
+<span class="sd"> Note: Because ``partition`` supports general logical operators, it</span>
+<span class="sd"> can be inefficient. Consider using NamedHivePartitionSensor instead if</span>
+<span class="sd"> you don't need the full flexibility of HivePartitionSensor.</span>
+
+<span class="sd"> :param table: The name of the table to wait for, supports the dot</span>
+<span class="sd"> notation (my_database.my_table)</span>
+<span class="sd"> :type table: string</span>
+<span class="sd"> :param partition: The partition clause to wait for. This is passed as</span>
+<span class="sd"> is to the metastore Thrift client ``get_partitions_by_filter`` method,</span>
+<span class="sd"> and apparently supports SQL like notation as in ``ds='2015-01-01'</span>
+<span class="sd"> AND type='value'`` and comparison operators as in ``"ds>=2015-01-01"``</span>
+<span class="sd"> :type partition: string</span>
+<span class="sd"> :param metastore_conn_id: reference to the metastore thrift service</span>
+<span class="sd"> connection id</span>
+<span class="sd"> :type metastore_conn_id: str</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'schema'</span><span class="p">,</span> <span class="s1">'table'</span><span class="p">,</span> <span class="s1">'partition'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#C5CAE9'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">table</span><span class="p">,</span> <span class="n">partition</span><span class="o">=</span><span class="s2">"ds='{{ ds }}'"</span><span class="p">,</span>
+ <span class="n">metastore_conn_id</span><span class="o">=</span><span class="s1">'metastore_default'</span><span class="p">,</span>
+ <span class="n">schema</span><span class="o">=</span><span class="s1">'default'</span><span class="p">,</span>
+ <span class="n">poke_interval</span><span class="o">=</span><span class="mi">60</span> <span class="o">*</span> <span class="mi">3</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HivePartitionSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+ <span class="n">poke_interval</span><span class="o">=</span><span class="n">poke_interval</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">partition</span><span class="p">:</span>
+ <span class="n">partition</span> <span class="o">=</span> <span class="s2">"ds='{{ ds }}'"</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span> <span class="o">=</span> <span class="n">metastore_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">table</span> <span class="o">=</span> <span class="n">table</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">partition</span> <span class="o">=</span> <span class="n">partition</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="o">=</span> <span class="n">schema</span>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="k">if</span> <span class="s1">'.'</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'Poking for table </span><span class="si">{self.schema}</span><span class="s1">.</span><span class="si">{self.table}</span><span class="s1">, '</span>
+ <span class="s1">'partition </span><span class="si">{self.partition}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">'hook'</span><span class="p">):</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="k">import</span> <span class="n">HiveMetastoreHook</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">(</span>
+ <span class="n">metastore_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">check_for_partition</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">partition</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/sensors/http_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/sensors/http_sensor.html b/_modules/airflow/sensors/http_sensor.html
new file mode 100644
index 0000000..d66f948
--- /dev/null
+++ b/_modules/airflow/sensors/http_sensor.html
@@ -0,0 +1,327 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.sensors.http_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.sensors.http_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.sensors.http_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.http_hook</span> <span class="k">import</span> <span class="n">HttpHook</span>
+<span class="kn">from</span> <span class="nn">airflow.sensors.base_sensor_operator</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="HttpSensor"><a class="viewcode-back" href="../../../code.html#airflow.sensors.http_sensor.HttpSensor">[docs]</a><span class="k">class</span> <span class="nc">HttpSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Executes a HTTP get statement and returns False on failure:</span>
+<span class="sd"> 404 not found or response_check function returned False</span>
+
+<span class="sd"> :param http_conn_id: The connection to run the sensor against</span>
+<span class="sd"> :type http_conn_id: string</span>
+<span class="sd"> :param method: The HTTP request method to use</span>
+<span class="sd"> :type method: string</span>
+<span class="sd"> :param endpoint: The relative part of the full url</span>
+<span class="sd"> :type endpoint: string</span>
+<span class="sd"> :param request_params: The parameters to be added to the GET url</span>
+<span class="sd"> :type request_params: a dictionary of string key/value pairs</span>
+<span class="sd"> :param headers: The HTTP headers to be added to the GET request</span>
+<span class="sd"> :type headers: a dictionary of string key/value pairs</span>
+<span class="sd"> :param response_check: A check against the 'requests' response object.</span>
+<span class="sd"> Returns True for 'pass' and False otherwise.</span>
+<span class="sd"> :type response_check: A lambda or defined function.</span>
+<span class="sd"> :param extra_options: Extra options for the 'requests' library, see the</span>
+<span class="sd"> 'requests' documentation (options to modify timeout, ssl, etc.)</span>
+<span class="sd"> :type extra_options: A dictionary of options, where key is string and value</span>
+<span class="sd"> depends on the option that's being modified.</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'endpoint'</span><span class="p">,</span> <span class="s1">'request_params'</span><span class="p">)</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">endpoint</span><span class="p">,</span>
+ <span class="n">http_conn_id</span><span class="o">=</span><span class="s1">'http_default'</span><span class="p">,</span>
+ <span class="n">method</span><span class="o">=</span><span class="s1">'GET'</span><span class="p">,</span>
+ <span class="n">request_params</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">headers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">response_check</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">extra_options</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HttpSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">endpoint</span> <span class="o">=</span> <span class="n">endpoint</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">http_conn_id</span> <span class="o">=</span> <span class="n">http_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">request_params</span> <span class="o">=</span> <span class="n">request_params</span> <span class="ow">or</span> <span class="p">{}</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">headers</span> <span class="o">=</span> <span class="n">headers</span> <span class="ow">or</span> <span class="p">{}</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">extra_options</span> <span class="o">=</span> <span class="n">extra_options</span> <span class="ow">or</span> <span class="p">{}</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">response_check</span> <span class="o">=</span> <span class="n">response_check</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">HttpHook</span><span class="p">(</span>
+ <span class="n">method</span><span class="o">=</span><span class="n">method</span><span class="p">,</span>
+ <span class="n">http_conn_id</span><span class="o">=</span><span class="n">http_conn_id</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Poking: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">endpoint</span><span class="p">)</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">endpoint</span><span class="p">,</span>
+ <span class="n">data</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">request_params</span><span class="p">,</span>
+ <span class="n">headers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">headers</span><span class="p">,</span>
+ <span class="n">extra_options</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">extra_options</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">response_check</span><span class="p">:</span>
+ <span class="c1"># run content check on response</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">response_check</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
+ <span class="k">except</span> <span class="n">AirflowException</span> <span class="k">as</span> <span class="n">ae</span><span class="p">:</span>
+ <span class="k">if</span> <span class="nb">str</span><span class="p">(</span><span class="n">ae</span><span class="p">)</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"404"</span><span class="p">):</span>
+ <span class="k">return</span> <span class="kc">False</span>
+
+ <span class="k">raise</span> <span class="n">ae</span>
+
+ <span class="k">return</span> <span class="kc">True</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/sensors/metastore_partition_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/sensors/metastore_partition_sensor.html b/_modules/airflow/sensors/metastore_partition_sensor.html
new file mode 100644
index 0000000..314c2af
--- /dev/null
+++ b/_modules/airflow/sensors/metastore_partition_sensor.html
@@ -0,0 +1,318 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.sensors.metastore_partition_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.sensors.metastore_partition_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.sensors.metastore_partition_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.sensors.sql_sensor</span> <span class="k">import</span> <span class="n">SqlSensor</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="MetastorePartitionSensor"><a class="viewcode-back" href="../../../code.html#airflow.sensors.metastore_partition_sensor.MetastorePartitionSensor">[docs]</a><span class="k">class</span> <span class="nc">MetastorePartitionSensor</span><span class="p">(</span><span class="n">SqlSensor</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> An alternative to the HivePartitionSensor that talk directly to the</span>
+<span class="sd"> MySQL db. This was created as a result of observing sub optimal</span>
+<span class="sd"> queries generated by the Metastore thrift service when hitting</span>
+<span class="sd"> subpartitioned tables. The Thrift service's queries were written in a</span>
+<span class="sd"> way that wouldn't leverage the indexes.</span>
+
+<span class="sd"> :param schema: the schema</span>
+<span class="sd"> :type schema: str</span>
+<span class="sd"> :param table: the table</span>
+<span class="sd"> :type table: str</span>
+<span class="sd"> :param partition_name: the partition name, as defined in the PARTITIONS</span>
+<span class="sd"> table of the Metastore. Order of the fields does matter.</span>
+<span class="sd"> Examples: ``ds=2016-01-01`` or</span>
+<span class="sd"> ``ds=2016-01-01/sub=foo`` for a sub partitioned table</span>
+<span class="sd"> :type partition_name: str</span>
+<span class="sd"> :param mysql_conn_id: a reference to the MySQL conn_id for the metastore</span>
+<span class="sd"> :type mysql_conn_id: str</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'partition_name'</span><span class="p">,</span> <span class="s1">'table'</span><span class="p">,</span> <span class="s1">'schema'</span><span class="p">)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#8da7be'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">table</span><span class="p">,</span>
+ <span class="n">partition_name</span><span class="p">,</span>
+ <span class="n">schema</span><span class="o">=</span><span class="s2">"default"</span><span class="p">,</span>
+ <span class="n">mysql_conn_id</span><span class="o">=</span><span class="s2">"metastore_mysql"</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">partition_name</span> <span class="o">=</span> <span class="n">partition_name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">table</span> <span class="o">=</span> <span class="n">table</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="o">=</span> <span class="n">schema</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">first_poke</span> <span class="o">=</span> <span class="kc">True</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span> <span class="o">=</span> <span class="n">mysql_conn_id</span>
+ <span class="c1"># TODO(aoen): We shouldn't be using SqlSensor here but MetastorePartitionSensor.</span>
+ <span class="c1"># The problem is the way apply_defaults works isn't compatible with inheritance.</span>
+ <span class="c1"># The inheritance model needs to be reworked in order to support overriding args/</span>
+ <span class="c1"># kwargs with arguments here, then 'conn_id' and 'sql' can be passed into the</span>
+ <span class="c1"># constructor below and apply_defaults will no longer throw an exception.</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">SqlSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">first_poke</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">first_poke</span> <span class="o">=</span> <span class="kc">False</span>
+ <span class="k">if</span> <span class="s1">'.'</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="s2">"""</span>
+<span class="s2"> SELECT 'X'</span>
+<span class="s2"> FROM PARTITIONS A0</span>
+<span class="s2"> LEFT OUTER JOIN TBLS B0 ON A0.TBL_ID = B0.TBL_ID</span>
+<span class="s2"> LEFT OUTER JOIN DBS C0 ON B0.DB_ID = C0.DB_ID</span>
+<span class="s2"> WHERE</span>
+<span class="s2"> B0.TBL_NAME = '</span><span class="si">{self.table}</span><span class="s2">' AND</span>
+<span class="s2"> C0.NAME = '</span><span class="si">{self.schema}</span><span class="s2">' AND</span>
+<span class="s2"> A0.PART_NAME = '</span><span class="si">{self.partition_name}</span><span class="s2">';</span>
+<span class="s2"> """</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">=</span><span class="bp">self</span><span class="p">)</span>
+ <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">MetastorePartitionSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">poke</span><span class="p">(</span><span class="n">context</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[35/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/bigquery_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/bigquery_operator.html b/_modules/airflow/contrib/operators/bigquery_operator.html
index 5a74ba1..4055654 100644
--- a/_modules/airflow/contrib/operators/bigquery_operator.html
+++ b/_modules/airflow/contrib/operators/bigquery_operator.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -169,19 +171,27 @@
<h1>Source code for airflow.contrib.operators.bigquery_operator</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">json</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="k">import</span> <span class="n">BigQueryHook</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcs_hook</span> <span class="k">import</span> <span class="n">GoogleCloudStorageHook</span><span class="p">,</span> <span class="n">_parse_gcs_url</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
@@ -204,6 +214,13 @@
<span class="sd"> :param create_disposition: Specifies whether the job is allowed to create new tables.</span>
<span class="sd"> (default: 'CREATE_IF_NEEDED')</span>
<span class="sd"> :type create_disposition: string</span>
+<span class="sd"> :param allow_large_results: Whether to allow large results.</span>
+<span class="sd"> :type allow_large_results: boolean</span>
+<span class="sd"> :param flatten_results: If true and query uses legacy SQL dialect, flattens</span>
+<span class="sd"> all nested and repeated fields in the query results. ``allow_large_results``</span>
+<span class="sd"> must be ``true`` if this is set to ``false``. For standard SQL queries, this</span>
+<span class="sd"> flag is ignored and results are never flattened.</span>
+<span class="sd"> :type flatten_results: boolean</span>
<span class="sd"> :param bigquery_conn_id: reference to a specific BigQuery hook.</span>
<span class="sd"> :type bigquery_conn_id: string</span>
<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
@@ -215,16 +232,25 @@
<span class="sd"> :type udf_config: list</span>
<span class="sd"> :param use_legacy_sql: Whether to use legacy SQL (true) or standard SQL (false).</span>
<span class="sd"> :type use_legacy_sql: boolean</span>
-<span class="sd"> :param maximum_billing_tier: Positive integer that serves as a multiplier of the basic price.</span>
+<span class="sd"> :param maximum_billing_tier: Positive integer that serves as a multiplier</span>
+<span class="sd"> of the basic price.</span>
<span class="sd"> Defaults to None, in which case it uses the value set in the project.</span>
<span class="sd"> :type maximum_billing_tier: integer</span>
-<span class="sd"> :param query_params: a dictionary containing query parameter types and values, passed to</span>
-<span class="sd"> BigQuery.</span>
+<span class="sd"> :param maximum_bytes_billed: Limits the bytes billed for this job.</span>
+<span class="sd"> Queries that will have bytes billed beyond this limit will fail</span>
+<span class="sd"> (without incurring a charge). If unspecified, this will be</span>
+<span class="sd"> set to your project default.</span>
+<span class="sd"> :type maximum_bytes_billed: float</span>
+<span class="sd"> :param schema_update_options: Allows the schema of the destination</span>
+<span class="sd"> table to be updated as a side effect of the load job.</span>
+<span class="sd"> :type schema_update_options: tuple</span>
+<span class="sd"> :param query_params: a dictionary containing query parameter types and</span>
+<span class="sd"> values, passed to BigQuery.</span>
<span class="sd"> :type query_params: dict</span>
<span class="sd"> """</span>
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bql'</span><span class="p">,</span> <span class="s1">'destination_dataset_table'</span><span class="p">)</span>
- <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,)</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,</span> <span class="p">)</span>
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#e4f0e8'</span>
<span class="nd">@apply_defaults</span>
@@ -233,13 +259,17 @@
<span class="n">destination_dataset_table</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">write_disposition</span><span class="o">=</span><span class="s1">'WRITE_EMPTY'</span><span class="p">,</span>
<span class="n">allow_large_results</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">flatten_results</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">'bigquery_default'</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">udf_config</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">use_legacy_sql</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="n">maximum_billing_tier</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">maximum_bytes_billed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">create_disposition</span><span class="o">=</span><span class="s1">'CREATE_IF_NEEDED'</span><span class="p">,</span>
+ <span class="n">schema_update_options</span><span class="o">=</span><span class="p">(),</span>
<span class="n">query_params</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">priority</span><span class="o">=</span><span class="s1">'INTERACTIVE'</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span>
<span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">BigQueryOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
@@ -248,23 +278,344 @@
<span class="bp">self</span><span class="o">.</span><span class="n">write_disposition</span> <span class="o">=</span> <span class="n">write_disposition</span>
<span class="bp">self</span><span class="o">.</span><span class="n">create_disposition</span> <span class="o">=</span> <span class="n">create_disposition</span>
<span class="bp">self</span><span class="o">.</span><span class="n">allow_large_results</span> <span class="o">=</span> <span class="n">allow_large_results</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">flatten_results</span> <span class="o">=</span> <span class="n">flatten_results</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
<span class="bp">self</span><span class="o">.</span><span class="n">udf_config</span> <span class="o">=</span> <span class="n">udf_config</span>
<span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span> <span class="o">=</span> <span class="n">use_legacy_sql</span>
<span class="bp">self</span><span class="o">.</span><span class="n">maximum_billing_tier</span> <span class="o">=</span> <span class="n">maximum_billing_tier</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">maximum_bytes_billed</span> <span class="o">=</span> <span class="n">maximum_bytes_billed</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">schema_update_options</span> <span class="o">=</span> <span class="n">schema_update_options</span>
<span class="bp">self</span><span class="o">.</span><span class="n">query_params</span> <span class="o">=</span> <span class="n">query_params</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bq_cursor</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">priority</span> <span class="o">=</span> <span class="n">priority</span>
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bql</span><span class="p">)</span>
- <span class="n">hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
- <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
- <span class="n">conn</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">bq_cursor</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bql</span><span class="p">)</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span>
+ <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
+ <span class="n">use_legacy_sql</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bq_cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bq_cursor</span><span class="o">.</span><span class="n">run_query</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bql</span><span class="p">,</span>
+ <span class="n">destination_dataset_table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">destination_dataset_table</span><span class="p">,</span>
+ <span class="n">write_disposition</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">write_disposition</span><span class="p">,</span>
+ <span class="n">allow_large_results</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">allow_large_results</span><span class="p">,</span>
+ <span class="n">flatten_results</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">flatten_results</span><span class="p">,</span>
+ <span class="n">udf_config</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">udf_config</span><span class="p">,</span>
+ <span class="n">maximum_billing_tier</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">maximum_billing_tier</span><span class="p">,</span>
+ <span class="n">maximum_bytes_billed</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">maximum_bytes_billed</span><span class="p">,</span>
+ <span class="n">create_disposition</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">create_disposition</span><span class="p">,</span>
+ <span class="n">query_params</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">query_params</span><span class="p">,</span>
+ <span class="n">schema_update_options</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">schema_update_options</span><span class="p">,</span>
+ <span class="n">priority</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">priority</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">on_kill</span><span class="p">()</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">bq_cursor</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Canceling running query due to execution timeout'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bq_cursor</span><span class="o">.</span><span class="n">cancel_query</span><span class="p">()</span></div>
+
+
+<div class="viewcode-block" id="BigQueryCreateEmptyTableOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_operator.BigQueryCreateEmptyTableOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryCreateEmptyTableOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Creates a new, empty table in the specified BigQuery dataset,</span>
+<span class="sd"> optionally with schema.</span>
+
+<span class="sd"> The schema to be used for the BigQuery table may be specified in one of</span>
+<span class="sd"> two ways. You may either directly pass the schema fields in, or you may</span>
+<span class="sd"> point the operator to a Google cloud storage object name. The object in</span>
+<span class="sd"> Google cloud storage must be a JSON file with the schema fields in it.</span>
+<span class="sd"> You can also create a table without schema.</span>
+
+<span class="sd"> :param project_id: The project to create the table into.</span>
+<span class="sd"> :type project_id: string</span>
+<span class="sd"> :param dataset_id: The dataset to create the table into.</span>
+<span class="sd"> :type dataset_id: string</span>
+<span class="sd"> :param table_id: The Name of the table to be created.</span>
+<span class="sd"> :type table_id: string</span>
+<span class="sd"> :param schema_fields: If set, the schema field list as defined here:</span>
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load.schema</span>
+
+<span class="sd"> **Example**: ::</span>
+
+<span class="sd"> schema_fields=[{"name": "emp_name", "type": "STRING", "mode": "REQUIRED"},</span>
+<span class="sd"> {"name": "salary", "type": "INTEGER", "mode": "NULLABLE"}]</span>
+
+<span class="sd"> :type schema_fields: list</span>
+<span class="sd"> :param gcs_schema_object: Full path to the JSON file containing schema. For</span>
+<span class="sd"> example: ``gs://test-bucket/dir1/dir2/employee_schema.json``</span>
+<span class="sd"> :type gcs_schema_object: string</span>
+<span class="sd"> :param time_partitioning: configure optional time partitioning fields i.e.</span>
+<span class="sd"> partition by field, type and expiration as per API specifications.</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#timePartitioning</span>
+<span class="sd"> :type time_partitioning: dict</span>
+<span class="sd"> :param bigquery_conn_id: Reference to a specific BigQuery hook.</span>
+<span class="sd"> :type bigquery_conn_id: string</span>
+<span class="sd"> :param google_cloud_storage_conn_id: Reference to a specific Google</span>
+<span class="sd"> cloud storage hook.</span>
+<span class="sd"> :type google_cloud_storage_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any. For this to</span>
+<span class="sd"> work, the service account making the request must have domain-wide</span>
+<span class="sd"> delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+
+<span class="sd"> **Example (with schema JSON in GCS)**: ::</span>
+
+<span class="sd"> CreateTable = BigQueryCreateEmptyTableOperator(</span>
+<span class="sd"> task_id='BigQueryCreateEmptyTableOperator_task',</span>
+<span class="sd"> dataset_id='ODS',</span>
+<span class="sd"> table_id='Employees',</span>
+<span class="sd"> project_id='internal-gcp-project',</span>
+<span class="sd"> gcs_schema_object='gs://schema-bucket/employee_schema.json',</span>
+<span class="sd"> bigquery_conn_id='airflow-service-account',</span>
+<span class="sd"> google_cloud_storage_conn_id='airflow-service-account'</span>
+<span class="sd"> )</span>
+
+<span class="sd"> **Corresponding Schema file** (``employee_schema.json``): ::</span>
+
+<span class="sd"> [</span>
+<span class="sd"> {</span>
+<span class="sd"> "mode": "NULLABLE",</span>
+<span class="sd"> "name": "emp_name",</span>
+<span class="sd"> "type": "STRING"</span>
+<span class="sd"> },</span>
+<span class="sd"> {</span>
+<span class="sd"> "mode": "REQUIRED",</span>
+<span class="sd"> "name": "salary",</span>
+<span class="sd"> "type": "INTEGER"</span>
+<span class="sd"> }</span>
+<span class="sd"> ]</span>
+
+<span class="sd"> **Example (with schema in the DAG)**: ::</span>
+
+<span class="sd"> CreateTable = BigQueryCreateEmptyTableOperator(</span>
+<span class="sd"> task_id='BigQueryCreateEmptyTableOperator_task',</span>
+<span class="sd"> dataset_id='ODS',</span>
+<span class="sd"> table_id='Employees',</span>
+<span class="sd"> project_id='internal-gcp-project',</span>
+<span class="sd"> schema_fields=[{"name": "emp_name", "type": "STRING", "mode": "REQUIRED"},</span>
+<span class="sd"> {"name": "salary", "type": "INTEGER", "mode": "NULLABLE"}],</span>
+<span class="sd"> bigquery_conn_id='airflow-service-account',</span>
+<span class="sd"> google_cloud_storage_conn_id='airflow-service-account'</span>
+<span class="sd"> )</span>
+
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'dataset_id'</span><span class="p">,</span> <span class="s1">'table_id'</span><span class="p">,</span> <span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'gcs_schema_object'</span><span class="p">)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f0eee4'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">dataset_id</span><span class="p">,</span>
+ <span class="n">table_id</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">schema_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">gcs_schema_object</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">time_partitioning</span><span class="o">=</span><span class="p">{},</span>
+ <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">'bigquery_default'</span><span class="p">,</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+
+ <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryCreateEmptyTableOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">dataset_id</span> <span class="o">=</span> <span class="n">dataset_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">table_id</span> <span class="o">=</span> <span class="n">table_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">schema_fields</span> <span class="o">=</span> <span class="n">schema_fields</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">gcs_schema_object</span> <span class="o">=</span> <span class="n">gcs_schema_object</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span> <span class="o">=</span> <span class="n">google_cloud_storage_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">time_partitioning</span> <span class="o">=</span> <span class="n">time_partitioning</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">bq_hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema_fields</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">gcs_schema_object</span><span class="p">:</span>
+
+ <span class="n">gcs_bucket</span><span class="p">,</span> <span class="n">gcs_object</span> <span class="o">=</span> <span class="n">_parse_gcs_url</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">gcs_schema_object</span><span class="p">)</span>
+
+ <span class="n">gcs_hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+ <span class="n">schema_fields</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">gcs_hook</span><span class="o">.</span><span class="n">download</span><span class="p">(</span>
+ <span class="n">gcs_bucket</span><span class="p">,</span>
+ <span class="n">gcs_object</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s2">"utf-8"</span><span class="p">))</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">schema_fields</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema_fields</span>
+
+ <span class="n">conn</span> <span class="o">=</span> <span class="n">bq_hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
+
+ <span class="n">cursor</span><span class="o">.</span><span class="n">create_empty_table</span><span class="p">(</span>
+ <span class="n">project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">dataset_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_id</span><span class="p">,</span>
+ <span class="n">table_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_id</span><span class="p">,</span>
+ <span class="n">schema_fields</span><span class="o">=</span><span class="n">schema_fields</span><span class="p">,</span>
+ <span class="n">time_partitioning</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">time_partitioning</span>
+ <span class="p">)</span></div>
+
+
+<div class="viewcode-block" id="BigQueryCreateExternalTableOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_operator.BigQueryCreateExternalTableOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryCreateExternalTableOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Creates a new external table in the dataset with the data in Google Cloud</span>
+<span class="sd"> Storage.</span>
+
+<span class="sd"> The schema to be used for the BigQuery table may be specified in one of</span>
+<span class="sd"> two ways. You may either directly pass the schema fields in, or you may</span>
+<span class="sd"> point the operator to a Google cloud storage object name. The object in</span>
+<span class="sd"> Google cloud storage must be a JSON file with the schema fields in it.</span>
+
+<span class="sd"> :param bucket: The bucket to point the external table to.</span>
+<span class="sd"> :type bucket: string</span>
+<span class="sd"> :param source_objects: List of Google cloud storage URIs to point table to.</span>
+<span class="sd"> If source_format is 'DATASTORE_BACKUP', the list must only contain a single URI.</span>
+<span class="sd"> :type object: list</span>
+<span class="sd"> :param destination_project_dataset_table: The dotted (<project>.)<dataset>.<table></span>
+<span class="sd"> BigQuery table to load data into. If <project> is not included, project will</span>
+<span class="sd"> be the project defined in the connection json.</span>
+<span class="sd"> :type destination_project_dataset_table: string</span>
+<span class="sd"> :param schema_fields: If set, the schema field list as defined here:</span>
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load.schema</span>
+
+<span class="sd"> **Example**: ::</span>
+
+<span class="sd"> schema_fields=[{"name": "emp_name", "type": "STRING", "mode": "REQUIRED"},</span>
+<span class="sd"> {"name": "salary", "type": "INTEGER", "mode": "NULLABLE"}]</span>
+
+<span class="sd"> Should not be set when source_format is 'DATASTORE_BACKUP'.</span>
+<span class="sd"> :type schema_fields: list</span>
+<span class="sd"> :param schema_object: If set, a GCS object path pointing to a .json file that</span>
+<span class="sd"> contains the schema for the table.</span>
+<span class="sd"> :param schema_object: string</span>
+<span class="sd"> :param source_format: File format of the data.</span>
+<span class="sd"> :type source_format: string</span>
+<span class="sd"> :param compression: [Optional] The compression type of the data source.</span>
+<span class="sd"> Possible values include GZIP and NONE.</span>
+<span class="sd"> The default value is NONE.</span>
+<span class="sd"> This setting is ignored for Google Cloud Bigtable,</span>
+<span class="sd"> Google Cloud Datastore backups and Avro formats.</span>
+<span class="sd"> :type compression: string</span>
+<span class="sd"> :param skip_leading_rows: Number of rows to skip when loading from a CSV.</span>
+<span class="sd"> :type skip_leading_rows: int</span>
+<span class="sd"> :param field_delimiter: The delimiter to use for the CSV.</span>
+<span class="sd"> :type field_delimiter: string</span>
+<span class="sd"> :param max_bad_records: The maximum number of bad records that BigQuery can</span>
+<span class="sd"> ignore when running the job.</span>
+<span class="sd"> :type max_bad_records: int</span>
+<span class="sd"> :param quote_character: The value that is used to quote data sections in a CSV file.</span>
+<span class="sd"> :type quote_character: string</span>
+<span class="sd"> :param allow_quoted_newlines: Whether to allow quoted newlines (true) or not (false).</span>
+<span class="sd"> :type allow_quoted_newlines: boolean</span>
+<span class="sd"> :param allow_jagged_rows: Accept rows that are missing trailing optional columns.</span>
+<span class="sd"> The missing values are treated as nulls. If false, records with missing trailing</span>
+<span class="sd"> columns are treated as bad records, and if there are too many bad records, an</span>
+<span class="sd"> invalid error is returned in the job result. Only applicable to CSV, ignored</span>
+<span class="sd"> for other formats.</span>
+<span class="sd"> :type allow_jagged_rows: bool</span>
+<span class="sd"> :param bigquery_conn_id: Reference to a specific BigQuery hook.</span>
+<span class="sd"> :type bigquery_conn_id: string</span>
+<span class="sd"> :param google_cloud_storage_conn_id: Reference to a specific Google</span>
+<span class="sd"> cloud storage hook.</span>
+<span class="sd"> :type google_cloud_storage_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any. For this to</span>
+<span class="sd"> work, the service account making the request must have domain-wide</span>
+<span class="sd"> delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> :param src_fmt_configs: configure optional fields specific to the source format</span>
+<span class="sd"> :type src_fmt_configs: dict</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bucket'</span><span class="p">,</span> <span class="s1">'source_objects'</span><span class="p">,</span>
+ <span class="s1">'schema_object'</span><span class="p">,</span> <span class="s1">'destination_project_dataset_table'</span><span class="p">)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f0eee4'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">bucket</span><span class="p">,</span>
+ <span class="n">source_objects</span><span class="p">,</span>
+ <span class="n">destination_project_dataset_table</span><span class="p">,</span>
+ <span class="n">schema_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">schema_object</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">source_format</span><span class="o">=</span><span class="s1">'CSV'</span><span class="p">,</span>
+ <span class="n">compression</span><span class="o">=</span><span class="s1">'NONE'</span><span class="p">,</span>
+ <span class="n">skip_leading_rows</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
+ <span class="n">field_delimiter</span><span class="o">=</span><span class="s1">','</span><span class="p">,</span>
+ <span class="n">max_bad_records</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
+ <span class="n">quote_character</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">allow_quoted_newlines</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">allow_jagged_rows</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">'bigquery_default'</span><span class="p">,</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">src_fmt_configs</span><span class="o">=</span><span class="p">{},</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+
+ <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryCreateExternalTableOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="c1"># GCS config</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">source_objects</span> <span class="o">=</span> <span class="n">source_objects</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">schema_object</span> <span class="o">=</span> <span class="n">schema_object</span>
+
+ <span class="c1"># BQ config</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">destination_project_dataset_table</span> <span class="o">=</span> <span class="n">destination_project_dataset_table</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">schema_fields</span> <span class="o">=</span> <span class="n">schema_fields</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">source_format</span> <span class="o">=</span> <span class="n">source_format</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">compression</span> <span class="o">=</span> <span class="n">compression</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">skip_leading_rows</span> <span class="o">=</span> <span class="n">skip_leading_rows</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">field_delimiter</span> <span class="o">=</span> <span class="n">field_delimiter</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">max_bad_records</span> <span class="o">=</span> <span class="n">max_bad_records</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">quote_character</span> <span class="o">=</span> <span class="n">quote_character</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">allow_quoted_newlines</span> <span class="o">=</span> <span class="n">allow_quoted_newlines</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">allow_jagged_rows</span> <span class="o">=</span> <span class="n">allow_jagged_rows</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span> <span class="o">=</span> <span class="n">google_cloud_storage_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">src_fmt_configs</span> <span class="o">=</span> <span class="n">src_fmt_configs</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">bq_hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema_fields</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema_object</span> \
+ <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">source_format</span> <span class="o">!=</span> <span class="s1">'DATASTORE_BACKUP'</span><span class="p">:</span>
+ <span class="n">gcs_hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+ <span class="n">schema_fields</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">gcs_hook</span><span class="o">.</span><span class="n">download</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">schema_object</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s2">"utf-8"</span><span class="p">))</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">schema_fields</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema_fields</span>
+
+ <span class="n">source_uris</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'gs://</span><span class="si">{}</span><span class="s1">/</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="n">source_object</span><span class="p">)</span>
+ <span class="k">for</span> <span class="n">source_object</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">source_objects</span><span class="p">]</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="n">bq_hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
<span class="n">cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
- <span class="n">cursor</span><span class="o">.</span><span class="n">run_query</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bql</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">destination_dataset_table</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">write_disposition</span><span class="p">,</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">allow_large_results</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">udf_config</span><span class="p">,</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">maximum_billing_tier</span><span class="p">,</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">create_disposition</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">query_params</span><span class="p">)</span></div>
+
+ <span class="n">cursor</span><span class="o">.</span><span class="n">create_external_table</span><span class="p">(</span>
+ <span class="n">external_project_dataset_table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">destination_project_dataset_table</span><span class="p">,</span>
+ <span class="n">schema_fields</span><span class="o">=</span><span class="n">schema_fields</span><span class="p">,</span>
+ <span class="n">source_uris</span><span class="o">=</span><span class="n">source_uris</span><span class="p">,</span>
+ <span class="n">source_format</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">source_format</span><span class="p">,</span>
+ <span class="n">compression</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">compression</span><span class="p">,</span>
+ <span class="n">skip_leading_rows</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">skip_leading_rows</span><span class="p">,</span>
+ <span class="n">field_delimiter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">field_delimiter</span><span class="p">,</span>
+ <span class="n">max_bad_records</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">max_bad_records</span><span class="p">,</span>
+ <span class="n">quote_character</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">quote_character</span><span class="p">,</span>
+ <span class="n">allow_quoted_newlines</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">allow_quoted_newlines</span><span class="p">,</span>
+ <span class="n">allow_jagged_rows</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">allow_jagged_rows</span><span class="p">,</span>
+ <span class="n">src_fmt_configs</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">src_fmt_configs</span>
+ <span class="p">)</span></div>
</pre></div>
</div>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/bigquery_table_delete_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/bigquery_table_delete_operator.html b/_modules/airflow/contrib/operators/bigquery_table_delete_operator.html
new file mode 100644
index 0000000..94bc9dd
--- /dev/null
+++ b/_modules/airflow/contrib/operators/bigquery_table_delete_operator.html
@@ -0,0 +1,301 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.bigquery_table_delete_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.bigquery_table_delete_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.bigquery_table_delete_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="k">import</span> <span class="n">BigQueryHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="BigQueryTableDeleteOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_table_delete_operator.BigQueryTableDeleteOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryTableDeleteOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Deletes BigQuery tables</span>
+
+<span class="sd"> :param deletion_dataset_table: A dotted</span>
+<span class="sd"> (<project>.|<project>:)<dataset>.<table> that indicates which table</span>
+<span class="sd"> will be deleted.</span>
+<span class="sd"> :type deletion_dataset_table: string</span>
+<span class="sd"> :param bigquery_conn_id: reference to a specific BigQuery hook.</span>
+<span class="sd"> :type bigquery_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request must have domain-wide</span>
+<span class="sd"> delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> :param ignore_if_missing: if True, then return success even if the</span>
+<span class="sd"> requested table does not exist.</span>
+<span class="sd"> :type ignore_if_missing: boolean</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'deletion_dataset_table'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#ffd1dc'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">deletion_dataset_table</span><span class="p">,</span>
+ <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">'bigquery_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">ignore_if_missing</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryTableDeleteOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">deletion_dataset_table</span> <span class="o">=</span> <span class="n">deletion_dataset_table</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">ignore_if_missing</span> <span class="o">=</span> <span class="n">ignore_if_missing</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Deleting: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">deletion_dataset_table</span><span class="p">)</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
+ <span class="n">cursor</span><span class="o">.</span><span class="n">run_table_delete</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">deletion_dataset_table</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ignore_if_missing</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/bigquery_to_bigquery.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/bigquery_to_bigquery.html b/_modules/airflow/contrib/operators/bigquery_to_bigquery.html
index ab12d51..0abb905 100644
--- a/_modules/airflow/contrib/operators/bigquery_to_bigquery.html
+++ b/_modules/airflow/contrib/operators/bigquery_to_bigquery.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,22 @@
<h1>Source code for airflow.contrib.operators.bigquery_to_bigquery</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="k">import</span> <span class="n">BigQueryHook</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
@@ -188,11 +195,11 @@
<div class="viewcode-block" id="BigQueryToBigQueryOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_to_bigquery.BigQueryToBigQueryOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryToBigQueryOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
<span class="sd">"""</span>
-<span class="sd"> Copies data from one BigQuery table to another. See here:</span>
-
-<span class="sd"> https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.copy</span>
+<span class="sd"> Copies data from one BigQuery table to another.</span>
-<span class="sd"> For more details about these parameters.</span>
+<span class="sd"> .. seealso::</span>
+<span class="sd"> For more details about these parameters:</span>
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.copy</span>
<span class="sd"> :param source_project_dataset_tables: One or more</span>
<span class="sd"> dotted (project:|project.)<dataset>.<table> BigQuery tables to use as the</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/bigquery_to_gcs.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/bigquery_to_gcs.html b/_modules/airflow/contrib/operators/bigquery_to_gcs.html
index 31355ff..6ebc523 100644
--- a/_modules/airflow/contrib/operators/bigquery_to_gcs.html
+++ b/_modules/airflow/contrib/operators/bigquery_to_gcs.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,22 @@
<h1>Source code for airflow.contrib.operators.bigquery_to_gcs</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="k">import</span> <span class="n">BigQueryHook</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
@@ -190,11 +197,9 @@
<span class="sd">"""</span>
<span class="sd"> Transfers a BigQuery table to a Google Cloud Storage bucket.</span>
-<span class="sd"> See here:</span>
-
-<span class="sd"> https://cloud.google.com/bigquery/docs/reference/v2/jobs</span>
-
-<span class="sd"> For more details about these parameters.</span>
+<span class="sd"> .. seealso::</span>
+<span class="sd"> For more details about these parameters:</span>
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/v2/jobs</span>
<span class="sd"> :param source_project_dataset_table: The dotted</span>
<span class="sd"> (<project>.|<project>:)<dataset>.<table> BigQuery table to use as the source</span>
[13/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/models.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/models.html b/_modules/airflow/models.html
index 485f9ae..7b01fed 100644
--- a/_modules/airflow/models.html
+++ b/_modules/airflow/models.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,22 @@
<h1>Source code for airflow.models</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="kn">from</span> <span class="nn">__future__</span> <span class="k">import</span> <span class="n">absolute_import</span>
<span class="kn">from</span> <span class="nn">__future__</span> <span class="k">import</span> <span class="n">division</span>
@@ -188,12 +195,12 @@
<span class="kn">from</span> <span class="nn">future.standard_library</span> <span class="k">import</span> <span class="n">install_aliases</span>
-<span class="n">install_aliases</span><span class="p">()</span>
<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">object</span><span class="p">,</span> <span class="nb">bytes</span>
<span class="kn">import</span> <span class="nn">copy</span>
-<span class="kn">from</span> <span class="nn">collections</span> <span class="k">import</span> <span class="n">namedtuple</span>
-<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
+<span class="kn">from</span> <span class="nn">collections</span> <span class="k">import</span> <span class="n">namedtuple</span><span class="p">,</span> <span class="n">defaultdict</span>
+<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">timedelta</span>
+
<span class="kn">import</span> <span class="nn">dill</span>
<span class="kn">import</span> <span class="nn">functools</span>
<span class="kn">import</span> <span class="nn">getpass</span>
@@ -208,21 +215,24 @@
<span class="kn">import</span> <span class="nn">pickle</span>
<span class="kn">import</span> <span class="nn">re</span>
<span class="kn">import</span> <span class="nn">signal</span>
-<span class="kn">import</span> <span class="nn">socket</span>
<span class="kn">import</span> <span class="nn">sys</span>
<span class="kn">import</span> <span class="nn">textwrap</span>
<span class="kn">import</span> <span class="nn">traceback</span>
<span class="kn">import</span> <span class="nn">warnings</span>
<span class="kn">import</span> <span class="nn">hashlib</span>
-<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="k">import</span> <span class="n">urlparse</span>
+
+<span class="kn">import</span> <span class="nn">uuid</span>
+<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span>
+<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="k">import</span> <span class="n">urlparse</span><span class="p">,</span> <span class="n">quote</span>
<span class="kn">from</span> <span class="nn">sqlalchemy</span> <span class="k">import</span> <span class="p">(</span>
<span class="n">Column</span><span class="p">,</span> <span class="n">Integer</span><span class="p">,</span> <span class="n">String</span><span class="p">,</span> <span class="n">DateTime</span><span class="p">,</span> <span class="n">Text</span><span class="p">,</span> <span class="n">Boolean</span><span class="p">,</span> <span class="n">ForeignKey</span><span class="p">,</span> <span class="n">PickleType</span><span class="p">,</span>
<span class="n">Index</span><span class="p">,</span> <span class="n">Float</span><span class="p">,</span> <span class="n">LargeBinary</span><span class="p">)</span>
-<span class="kn">from</span> <span class="nn">sqlalchemy</span> <span class="k">import</span> <span class="n">func</span><span class="p">,</span> <span class="n">or_</span><span class="p">,</span> <span class="n">and_</span>
+<span class="kn">from</span> <span class="nn">sqlalchemy</span> <span class="k">import</span> <span class="n">func</span><span class="p">,</span> <span class="n">or_</span><span class="p">,</span> <span class="n">and_</span><span class="p">,</span> <span class="n">true</span> <span class="k">as</span> <span class="n">sqltrue</span>
<span class="kn">from</span> <span class="nn">sqlalchemy.ext.declarative</span> <span class="k">import</span> <span class="n">declarative_base</span><span class="p">,</span> <span class="n">declared_attr</span>
<span class="kn">from</span> <span class="nn">sqlalchemy.dialects.mysql</span> <span class="k">import</span> <span class="n">LONGTEXT</span>
<span class="kn">from</span> <span class="nn">sqlalchemy.orm</span> <span class="k">import</span> <span class="n">reconstructor</span><span class="p">,</span> <span class="n">relationship</span><span class="p">,</span> <span class="n">synonym</span>
+<span class="kn">from</span> <span class="nn">sqlalchemy_utc</span> <span class="k">import</span> <span class="n">UtcDateTime</span>
<span class="kn">from</span> <span class="nn">croniter</span> <span class="k">import</span> <span class="n">croniter</span>
<span class="kn">import</span> <span class="nn">six</span>
@@ -230,14 +240,18 @@
<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">settings</span><span class="p">,</span> <span class="n">utils</span>
<span class="kn">from</span> <span class="nn">airflow.executors</span> <span class="k">import</span> <span class="n">GetDefaultExecutor</span><span class="p">,</span> <span class="n">LocalExecutor</span>
<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">configuration</span>
-<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span><span class="p">,</span> <span class="n">AirflowSkipException</span><span class="p">,</span> <span class="n">AirflowTaskTimeout</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="p">(</span>
+ <span class="n">AirflowDagCycleException</span><span class="p">,</span> <span class="n">AirflowException</span><span class="p">,</span> <span class="n">AirflowSkipException</span><span class="p">,</span> <span class="n">AirflowTaskTimeout</span>
+<span class="p">)</span>
<span class="kn">from</span> <span class="nn">airflow.dag.base_dag</span> <span class="k">import</span> <span class="n">BaseDag</span><span class="p">,</span> <span class="n">BaseDagBag</span>
+<span class="kn">from</span> <span class="nn">airflow.lineage</span> <span class="k">import</span> <span class="n">apply_lineage</span><span class="p">,</span> <span class="n">prepare_lineage</span>
<span class="kn">from</span> <span class="nn">airflow.ti_deps.deps.not_in_retry_period_dep</span> <span class="k">import</span> <span class="n">NotInRetryPeriodDep</span>
<span class="kn">from</span> <span class="nn">airflow.ti_deps.deps.prev_dagrun_dep</span> <span class="k">import</span> <span class="n">PrevDagrunDep</span>
<span class="kn">from</span> <span class="nn">airflow.ti_deps.deps.trigger_rule_dep</span> <span class="k">import</span> <span class="n">TriggerRuleDep</span>
<span class="kn">from</span> <span class="nn">airflow.ti_deps.deps.task_concurrency_dep</span> <span class="k">import</span> <span class="n">TaskConcurrencyDep</span>
<span class="kn">from</span> <span class="nn">airflow.ti_deps.dep_context</span> <span class="k">import</span> <span class="n">DepContext</span><span class="p">,</span> <span class="n">QUEUE_DEPS</span><span class="p">,</span> <span class="n">RUN_DEPS</span>
+<span class="kn">from</span> <span class="nn">airflow.utils</span> <span class="k">import</span> <span class="n">timezone</span>
<span class="kn">from</span> <span class="nn">airflow.utils.dates</span> <span class="k">import</span> <span class="n">cron_presets</span><span class="p">,</span> <span class="n">date_range</span> <span class="k">as</span> <span class="n">utils_date_range</span>
<span class="kn">from</span> <span class="nn">airflow.utils.db</span> <span class="k">import</span> <span class="n">provide_session</span>
<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
@@ -248,15 +262,19 @@
<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span>
<span class="kn">from</span> <span class="nn">airflow.utils.timeout</span> <span class="k">import</span> <span class="n">timeout</span>
<span class="kn">from</span> <span class="nn">airflow.utils.trigger_rule</span> <span class="k">import</span> <span class="n">TriggerRule</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.weight_rule</span> <span class="k">import</span> <span class="n">WeightRule</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.net</span> <span class="k">import</span> <span class="n">get_hostname</span>
<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+<span class="n">install_aliases</span><span class="p">()</span>
+
<span class="n">Base</span> <span class="o">=</span> <span class="n">declarative_base</span><span class="p">()</span>
<span class="n">ID_LEN</span> <span class="o">=</span> <span class="mi">250</span>
<span class="n">XCOM_RETURN_KEY</span> <span class="o">=</span> <span class="s1">'return_value'</span>
<span class="n">Stats</span> <span class="o">=</span> <span class="n">settings</span><span class="o">.</span><span class="n">Stats</span>
-<span class="k">def</span> <span class="nf">get_fernet</span><span class="p">():</span>
+<div class="viewcode-block" id="get_fernet"><a class="viewcode-back" href="../../code.html#airflow.models.get_fernet">[docs]</a><span class="k">def</span> <span class="nf">get_fernet</span><span class="p">():</span>
<span class="sd">"""</span>
<span class="sd"> Deferred load of Fernet key.</span>
@@ -271,21 +289,15 @@
<span class="k">except</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Failed to import Fernet, it may not be installed'</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">Fernet</span><span class="p">(</span><span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'FERNET_KEY'</span><span class="p">)</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">))</span>
- <span class="k">except</span> <span class="ne">ValueError</span> <span class="k">as</span> <span class="n">ve</span><span class="p">:</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Could not create Fernet object: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">ve</span><span class="p">))</span>
-
+ <span class="k">return</span> <span class="n">Fernet</span><span class="p">(</span><span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'FERNET_KEY'</span><span class="p">)</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">))</span>
+ <span class="k">except</span> <span class="p">(</span><span class="ne">ValueError</span><span class="p">,</span> <span class="ne">TypeError</span><span class="p">)</span> <span class="k">as</span> <span class="n">ve</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Could not create Fernet object: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">ve</span><span class="p">))</span></div>
-<span class="k">if</span> <span class="s1">'mysql'</span> <span class="ow">in</span> <span class="n">settings</span><span class="o">.</span><span class="n">SQL_ALCHEMY_CONN</span><span class="p">:</span>
- <span class="n">LongText</span> <span class="o">=</span> <span class="n">LONGTEXT</span>
-<span class="k">else</span><span class="p">:</span>
- <span class="n">LongText</span> <span class="o">=</span> <span class="n">Text</span>
<span class="c1"># Used by DAG context_managers</span>
<span class="n">_CONTEXT_MANAGER_DAG</span> <span class="o">=</span> <span class="kc">None</span>
-
-<span class="k">def</span> <span class="nf">clear_task_instances</span><span class="p">(</span><span class="n">tis</span><span class="p">,</span> <span class="n">session</span><span class="p">,</span> <span class="n">activate_dag_runs</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">dag</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+<div class="viewcode-block" id="clear_task_instances"><a class="viewcode-back" href="../../code.html#airflow.models.clear_task_instances">[docs]</a><span class="k">def</span> <span class="nf">clear_task_instances</span><span class="p">(</span><span class="n">tis</span><span class="p">,</span> <span class="n">session</span><span class="p">,</span> <span class="n">activate_dag_runs</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">dag</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Clears a set of task instances, but makes sure the running ones</span>
<span class="sd"> get killed.</span>
@@ -323,7 +335,7 @@
<span class="p">)</span><span class="o">.</span><span class="n">all</span><span class="p">()</span>
<span class="k">for</span> <span class="n">dr</span> <span class="ow">in</span> <span class="n">drs</span><span class="p">:</span>
<span class="n">dr</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span>
- <span class="n">dr</span><span class="o">.</span><span class="n">start_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
+ <span class="n">dr</span><span class="o">.</span><span class="n">start_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span></div>
<div class="viewcode-block" id="DagBag"><a class="viewcode-back" href="../../code.html#airflow.models.DagBag">[docs]</a><span class="k">class</span> <span class="nc">DagBag</span><span class="p">(</span><span class="n">BaseDagBag</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
@@ -345,11 +357,16 @@
<span class="sd"> :type include_examples: bool</span>
<span class="sd"> """</span>
+ <span class="c1"># static class variables to detetct dag cycle</span>
+ <span class="n">CYCLE_NEW</span> <span class="o">=</span> <span class="mi">0</span>
+ <span class="n">CYCLE_IN_PROGRESS</span> <span class="o">=</span> <span class="mi">1</span>
+ <span class="n">CYCLE_DONE</span> <span class="o">=</span> <span class="mi">2</span>
+
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">dag_folder</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">executor</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">include_examples</span><span class="o">=</span><span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'LOAD_EXAMPLES'</span><span class="p">)):</span>
+ <span class="n">include_examples</span><span class="o">=</span><span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'LOAD_EXAMPLES'</span><span class="p">)):</span>
<span class="c1"># do not use default arg in signature, to fix import cycle on plugin load</span>
<span class="k">if</span> <span class="n">executor</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
@@ -414,8 +431,10 @@
<span class="sd"> """</span>
<span class="n">found_dags</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="c1"># if the source file no longer exists in the DB or in the filesystem,</span>
+ <span class="c1"># return an empty list</span>
<span class="c1"># todo: raise exception?</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">filepath</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">filepath</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">filepath</span><span class="p">):</span>
<span class="k">return</span> <span class="n">found_dags</span>
<span class="k">try</span><span class="p">:</span>
@@ -449,7 +468,7 @@
<span class="k">if</span> <span class="n">mod_name</span> <span class="ow">in</span> <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="p">:</span>
<span class="k">del</span> <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="p">[</span><span class="n">mod_name</span><span class="p">]</span>
- <span class="k">with</span> <span class="n">timeout</span><span class="p">(</span><span class="n">configuration</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s2">"DAGBAG_IMPORT_TIMEOUT"</span><span class="p">)):</span>
+ <span class="k">with</span> <span class="n">timeout</span><span class="p">(</span><span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s2">"DAGBAG_IMPORT_TIMEOUT"</span><span class="p">)):</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">m</span> <span class="o">=</span> <span class="n">imp</span><span class="o">.</span><span class="n">load_source</span><span class="p">(</span><span class="n">mod_name</span><span class="p">,</span> <span class="n">filepath</span><span class="p">)</span>
<span class="n">mods</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">m</span><span class="p">)</span>
@@ -493,10 +512,19 @@
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dag</span><span class="p">,</span> <span class="n">DAG</span><span class="p">):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span><span class="p">:</span>
<span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span> <span class="o">=</span> <span class="n">filepath</span>
- <span class="n">dag</span><span class="o">.</span><span class="n">is_subdag</span> <span class="o">=</span> <span class="kc">False</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">bag_dag</span><span class="p">(</span><span class="n">dag</span><span class="p">,</span> <span class="n">parent_dag</span><span class="o">=</span><span class="n">dag</span><span class="p">,</span> <span class="n">root_dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span>
- <span class="n">found_dags</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">dag</span><span class="p">)</span>
- <span class="n">found_dags</span> <span class="o">+=</span> <span class="n">dag</span><span class="o">.</span><span class="n">subdags</span>
+ <span class="k">if</span> <span class="n">dag</span><span class="o">.</span><span class="n">fileloc</span> <span class="o">!=</span> <span class="n">filepath</span><span class="p">:</span>
+ <span class="n">dag</span><span class="o">.</span><span class="n">fileloc</span> <span class="o">=</span> <span class="n">filepath</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">dag</span><span class="o">.</span><span class="n">is_subdag</span> <span class="o">=</span> <span class="kc">False</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bag_dag</span><span class="p">(</span><span class="n">dag</span><span class="p">,</span> <span class="n">parent_dag</span><span class="o">=</span><span class="n">dag</span><span class="p">,</span> <span class="n">root_dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span>
+ <span class="n">found_dags</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">dag</span><span class="p">)</span>
+ <span class="n">found_dags</span> <span class="o">+=</span> <span class="n">dag</span><span class="o">.</span><span class="n">subdags</span>
+ <span class="k">except</span> <span class="n">AirflowDagCycleException</span> <span class="k">as</span> <span class="n">cycle_exception</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s2">"Failed to bag_dag: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">import_errors</span><span class="p">[</span><span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">cycle_exception</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">file_last_changed</span><span class="p">[</span><span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span><span class="p">]</span> <span class="o">=</span> \
+ <span class="n">file_last_changed_on_disk</span>
+
<span class="bp">self</span><span class="o">.</span><span class="n">file_last_changed</span><span class="p">[</span><span class="n">filepath</span><span class="p">]</span> <span class="o">=</span> <span class="n">file_last_changed_on_disk</span>
<span class="k">return</span> <span class="n">found_dags</span></div>
@@ -509,8 +537,8 @@
<span class="kn">from</span> <span class="nn">airflow.jobs</span> <span class="k">import</span> <span class="n">LocalTaskJob</span> <span class="k">as</span> <span class="n">LJ</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Finding 'running' jobs without a recent heartbeat"</span><span class="p">)</span>
<span class="n">TI</span> <span class="o">=</span> <span class="n">TaskInstance</span>
- <span class="n">secs</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'scheduler'</span><span class="p">,</span> <span class="s1">'scheduler_zombie_task_threshold'</span><span class="p">)</span>
- <span class="n">limit_dttm</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">seconds</span><span class="o">=</span><span class="n">secs</span><span class="p">)</span>
+ <span class="n">secs</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'scheduler'</span><span class="p">,</span> <span class="s1">'scheduler_zombie_task_threshold'</span><span class="p">)</span>
+ <span class="n">limit_dttm</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">seconds</span><span class="o">=</span><span class="n">secs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Failing jobs without heartbeat after </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">limit_dttm</span><span class="p">)</span>
<span class="n">tis</span> <span class="o">=</span> <span class="p">(</span>
@@ -539,20 +567,39 @@
<div class="viewcode-block" id="DagBag.bag_dag"><a class="viewcode-back" href="../../code.html#airflow.models.DagBag.bag_dag">[docs]</a> <span class="k">def</span> <span class="nf">bag_dag</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dag</span><span class="p">,</span> <span class="n">parent_dag</span><span class="p">,</span> <span class="n">root_dag</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Adds the DAG into the bag, recurses into sub dags.</span>
+<span class="sd"> Throws AirflowDagCycleException if a cycle is detected in this dag or its subdags</span>
<span class="sd"> """</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">dags</span><span class="p">[</span><span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">]</span> <span class="o">=</span> <span class="n">dag</span>
+
+ <span class="n">dag</span><span class="o">.</span><span class="n">test_cycle</span><span class="p">()</span> <span class="c1"># throws if a task cycle is found</span>
+
<span class="n">dag</span><span class="o">.</span><span class="n">resolve_template_files</span><span class="p">()</span>
- <span class="n">dag</span><span class="o">.</span><span class="n">last_loaded</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
+ <span class="n">dag</span><span class="o">.</span><span class="n">last_loaded</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
<span class="k">for</span> <span class="n">task</span> <span class="ow">in</span> <span class="n">dag</span><span class="o">.</span><span class="n">tasks</span><span class="p">:</span>
<span class="n">settings</span><span class="o">.</span><span class="n">policy</span><span class="p">(</span><span class="n">task</span><span class="p">)</span>
- <span class="k">for</span> <span class="n">subdag</span> <span class="ow">in</span> <span class="n">dag</span><span class="o">.</span><span class="n">subdags</span><span class="p">:</span>
- <span class="n">subdag</span><span class="o">.</span><span class="n">full_filepath</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span>
- <span class="n">subdag</span><span class="o">.</span><span class="n">parent_dag</span> <span class="o">=</span> <span class="n">dag</span>
- <span class="n">subdag</span><span class="o">.</span><span class="n">is_subdag</span> <span class="o">=</span> <span class="kc">True</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">bag_dag</span><span class="p">(</span><span class="n">subdag</span><span class="p">,</span> <span class="n">parent_dag</span><span class="o">=</span><span class="n">dag</span><span class="p">,</span> <span class="n">root_dag</span><span class="o">=</span><span class="n">root_dag</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'Loaded DAG </span><span class="si">{dag}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span></div>
+ <span class="n">subdags</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">subdags</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="k">for</span> <span class="n">subdag</span> <span class="ow">in</span> <span class="n">subdags</span><span class="p">:</span>
+ <span class="n">subdag</span><span class="o">.</span><span class="n">full_filepath</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span>
+ <span class="n">subdag</span><span class="o">.</span><span class="n">parent_dag</span> <span class="o">=</span> <span class="n">dag</span>
+ <span class="n">subdag</span><span class="o">.</span><span class="n">is_subdag</span> <span class="o">=</span> <span class="kc">True</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bag_dag</span><span class="p">(</span><span class="n">subdag</span><span class="p">,</span> <span class="n">parent_dag</span><span class="o">=</span><span class="n">dag</span><span class="p">,</span> <span class="n">root_dag</span><span class="o">=</span><span class="n">root_dag</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">dags</span><span class="p">[</span><span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">]</span> <span class="o">=</span> <span class="n">dag</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'Loaded DAG </span><span class="si">{dag}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
+ <span class="k">except</span> <span class="n">AirflowDagCycleException</span> <span class="k">as</span> <span class="n">cycle_exception</span><span class="p">:</span>
+ <span class="c1"># There was an error in bagging the dag. Remove it from the list of dags</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s1">'Exception bagging dag: </span><span class="si">{dag.dag_id}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
+ <span class="c1"># Only necessary at the root level since DAG.subdags automatically</span>
+ <span class="c1"># performs DFS to search through all subdags</span>
+ <span class="k">if</span> <span class="n">dag</span> <span class="o">==</span> <span class="n">root_dag</span><span class="p">:</span>
+ <span class="k">for</span> <span class="n">subdag</span> <span class="ow">in</span> <span class="n">subdags</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">subdag</span><span class="o">.</span><span class="n">dag_id</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags</span><span class="p">:</span>
+ <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags</span><span class="p">[</span><span class="n">subdag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">]</span>
+ <span class="k">raise</span> <span class="n">cycle_exception</span></div>
+
<div class="viewcode-block" id="DagBag.collect_dags"><a class="viewcode-back" href="../../code.html#airflow.models.DagBag.collect_dags">[docs]</a> <span class="k">def</span> <span class="nf">collect_dags</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
@@ -567,7 +614,7 @@
<span class="sd"> ignoring files that match any of the regex patterns specified</span>
<span class="sd"> in the file.</span>
<span class="sd"> """</span>
- <span class="n">start_dttm</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
+ <span class="n">start_dttm</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
<span class="n">dag_folder</span> <span class="o">=</span> <span class="n">dag_folder</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_folder</span>
<span class="c1"># Used to store stats around DagBag processing</span>
@@ -577,13 +624,12 @@
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">dag_folder</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">process_file</span><span class="p">(</span><span class="n">dag_folder</span><span class="p">,</span> <span class="n">only_if_updated</span><span class="o">=</span><span class="n">only_if_updated</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isdir</span><span class="p">(</span><span class="n">dag_folder</span><span class="p">):</span>
- <span class="n">patterns</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">root</span><span class="p">,</span> <span class="n">dirs</span><span class="p">,</span> <span class="n">files</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">walk</span><span class="p">(</span><span class="n">dag_folder</span><span class="p">,</span> <span class="n">followlinks</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
- <span class="n">ignore_file</span> <span class="o">=</span> <span class="p">[</span><span class="n">f</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">files</span> <span class="k">if</span> <span class="n">f</span> <span class="o">==</span> <span class="s1">'.airflowignore'</span><span class="p">]</span>
- <span class="k">if</span> <span class="n">ignore_file</span><span class="p">:</span>
- <span class="n">f</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">root</span><span class="p">,</span> <span class="n">ignore_file</span><span class="p">[</span><span class="mi">0</span><span class="p">]),</span> <span class="s1">'r'</span><span class="p">)</span>
- <span class="n">patterns</span> <span class="o">+=</span> <span class="p">[</span><span class="n">p</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">)</span> <span class="k">if</span> <span class="n">p</span><span class="p">]</span>
- <span class="n">f</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
+ <span class="n">patterns</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="n">ignore_file</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">root</span><span class="p">,</span> <span class="s1">'.airflowignore'</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">ignore_file</span><span class="p">):</span>
+ <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">ignore_file</span><span class="p">,</span> <span class="s1">'r'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
+ <span class="n">patterns</span> <span class="o">+=</span> <span class="p">[</span><span class="n">p</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">)</span> <span class="k">if</span> <span class="n">p</span><span class="p">]</span>
<span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">files</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">filepath</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">root</span><span class="p">,</span> <span class="n">f</span><span class="p">)</span>
@@ -595,11 +641,11 @@
<span class="k">continue</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">any</span><span class="p">(</span>
<span class="p">[</span><span class="n">re</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="n">p</span><span class="p">,</span> <span class="n">filepath</span><span class="p">)</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">patterns</span><span class="p">]):</span>
- <span class="n">ts</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
+ <span class="n">ts</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
<span class="n">found_dags</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">process_file</span><span class="p">(</span>
<span class="n">filepath</span><span class="p">,</span> <span class="n">only_if_updated</span><span class="o">=</span><span class="n">only_if_updated</span><span class="p">)</span>
- <span class="n">td</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">-</span> <span class="n">ts</span>
+ <span class="n">td</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">-</span> <span class="n">ts</span>
<span class="n">td</span> <span class="o">=</span> <span class="n">td</span><span class="o">.</span><span class="n">total_seconds</span><span class="p">()</span> <span class="o">+</span> <span class="p">(</span>
<span class="nb">float</span><span class="p">(</span><span class="n">td</span><span class="o">.</span><span class="n">microseconds</span><span class="p">)</span> <span class="o">/</span> <span class="mi">1000000</span><span class="p">)</span>
<span class="n">stats</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">FileLoadStat</span><span class="p">(</span>
@@ -610,9 +656,9 @@
<span class="nb">str</span><span class="p">([</span><span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span> <span class="k">for</span> <span class="n">dag</span> <span class="ow">in</span> <span class="n">found_dags</span><span class="p">]),</span>
<span class="p">))</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
<span class="n">Stats</span><span class="o">.</span><span class="n">gauge</span><span class="p">(</span>
- <span class="s1">'collect_dags'</span><span class="p">,</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">-</span> <span class="n">start_dttm</span><span class="p">)</span><span class="o">.</span><span class="n">total_seconds</span><span class="p">(),</span> <span class="mi">1</span><span class="p">)</span>
+ <span class="s1">'collect_dags'</span><span class="p">,</span> <span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">-</span> <span class="n">start_dttm</span><span class="p">)</span><span class="o">.</span><span class="n">total_seconds</span><span class="p">(),</span> <span class="mi">1</span><span class="p">)</span>
<span class="n">Stats</span><span class="o">.</span><span class="n">gauge</span><span class="p">(</span>
<span class="s1">'dagbag_size'</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dags</span><span class="p">),</span> <span class="mi">1</span><span class="p">)</span>
<span class="n">Stats</span><span class="o">.</span><span class="n">gauge</span><span class="p">(</span>
@@ -636,26 +682,23 @@
<span class="n">dag_folder</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_folder</span><span class="p">,</span>
<span class="n">duration</span><span class="o">=</span><span class="nb">sum</span><span class="p">([</span><span class="n">o</span><span class="o">.</span><span class="n">duration</span> <span class="k">for</span> <span class="n">o</span> <span class="ow">in</span> <span class="n">stats</span><span class="p">]),</span>
<span class="n">dag_num</span><span class="o">=</span><span class="nb">sum</span><span class="p">([</span><span class="n">o</span><span class="o">.</span><span class="n">dag_num</span> <span class="k">for</span> <span class="n">o</span> <span class="ow">in</span> <span class="n">stats</span><span class="p">]),</span>
- <span class="n">task_num</span><span class="o">=</span><span class="nb">sum</span><span class="p">([</span><span class="n">o</span><span class="o">.</span><span class="n">dag_num</span> <span class="k">for</span> <span class="n">o</span> <span class="ow">in</span> <span class="n">stats</span><span class="p">]),</span>
+ <span class="n">task_num</span><span class="o">=</span><span class="nb">sum</span><span class="p">([</span><span class="n">o</span><span class="o">.</span><span class="n">task_num</span> <span class="k">for</span> <span class="n">o</span> <span class="ow">in</span> <span class="n">stats</span><span class="p">]),</span>
<span class="n">table</span><span class="o">=</span><span class="n">pprinttable</span><span class="p">(</span><span class="n">stats</span><span class="p">),</span>
<span class="p">)</span></div>
- <span class="k">def</span> <span class="nf">deactivate_inactive_dags</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="nd">@provide_session</span>
+ <span class="k">def</span> <span class="nf">deactivate_inactive_dags</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="n">active_dag_ids</span> <span class="o">=</span> <span class="p">[</span><span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span> <span class="k">for</span> <span class="n">dag</span> <span class="ow">in</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dags</span><span class="o">.</span><span class="n">values</span><span class="p">())]</span>
- <span class="n">session</span> <span class="o">=</span> <span class="n">settings</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span>
<span class="k">for</span> <span class="n">dag</span> <span class="ow">in</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span>
<span class="n">DagModel</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="o">~</span><span class="n">DagModel</span><span class="o">.</span><span class="n">dag_id</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">active_dag_ids</span><span class="p">))</span><span class="o">.</span><span class="n">all</span><span class="p">():</span>
<span class="n">dag</span><span class="o">.</span><span class="n">is_active</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">session</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="n">dag</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
- <span class="n">session</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
- <span class="k">def</span> <span class="nf">paused_dags</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="n">session</span> <span class="o">=</span> <span class="n">settings</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span>
+ <span class="nd">@provide_session</span>
+ <span class="k">def</span> <span class="nf">paused_dags</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="n">dag_ids</span> <span class="o">=</span> <span class="p">[</span><span class="n">dp</span><span class="o">.</span><span class="n">dag_id</span> <span class="k">for</span> <span class="n">dp</span> <span class="ow">in</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagModel</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
<span class="n">DagModel</span><span class="o">.</span><span class="n">is_paused</span><span class="o">.</span><span class="fm">__eq__</span><span class="p">(</span><span class="kc">True</span><span class="p">))]</span>
- <span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
- <span class="n">session</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
<span class="k">return</span> <span class="n">dag_ids</span></div>
@@ -709,6 +752,7 @@
<span class="p">(</span><span class="s1">'hive_metastore'</span><span class="p">,</span> <span class="s1">'Hive Metastore Thrift'</span><span class="p">,),</span>
<span class="p">(</span><span class="s1">'hiveserver2'</span><span class="p">,</span> <span class="s1">'Hive Server 2 Thrift'</span><span class="p">,),</span>
<span class="p">(</span><span class="s1">'jdbc'</span><span class="p">,</span> <span class="s1">'Jdbc Connection'</span><span class="p">,),</span>
+ <span class="p">(</span><span class="s1">'jenkins'</span><span class="p">,</span> <span class="s1">'Jenkins'</span><span class="p">),</span>
<span class="p">(</span><span class="s1">'mysql'</span><span class="p">,</span> <span class="s1">'MySQL'</span><span class="p">,),</span>
<span class="p">(</span><span class="s1">'postgres'</span><span class="p">,</span> <span class="s1">'Postgres'</span><span class="p">,),</span>
<span class="p">(</span><span class="s1">'oracle'</span><span class="p">,</span> <span class="s1">'Oracle'</span><span class="p">,),</span>
@@ -727,6 +771,8 @@
<span class="p">(</span><span class="s1">'databricks'</span><span class="p">,</span> <span class="s1">'Databricks'</span><span class="p">,),</span>
<span class="p">(</span><span class="s1">'aws'</span><span class="p">,</span> <span class="s1">'Amazon Web Services'</span><span class="p">,),</span>
<span class="p">(</span><span class="s1">'emr'</span><span class="p">,</span> <span class="s1">'Elastic MapReduce'</span><span class="p">,),</span>
+ <span class="p">(</span><span class="s1">'snowflake'</span><span class="p">,</span> <span class="s1">'Snowflake'</span><span class="p">,),</span>
+ <span class="p">(</span><span class="s1">'segment'</span><span class="p">,</span> <span class="s1">'Segment'</span><span class="p">,),</span>
<span class="p">]</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
@@ -813,6 +859,9 @@
<span class="s2">"using non-encrypted value."</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_extra</span> <span class="o">=</span> <span class="n">value</span>
<span class="bp">self</span><span class="o">.</span><span class="n">is_extra_encrypted</span> <span class="o">=</span> <span class="kc">False</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_extra</span> <span class="o">=</span> <span class="n">value</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">is_extra_encrypted</span> <span class="o">=</span> <span class="kc">False</span>
<span class="nd">@declared_attr</span>
<span class="k">def</span> <span class="nf">extra</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
@@ -889,7 +938,7 @@
<span class="k">return</span> <span class="n">obj</span></div>
-<span class="k">class</span> <span class="nc">DagPickle</span><span class="p">(</span><span class="n">Base</span><span class="p">):</span>
+<div class="viewcode-block" id="DagPickle"><a class="viewcode-back" href="../../code.html#airflow.models.DagPickle">[docs]</a><span class="k">class</span> <span class="nc">DagPickle</span><span class="p">(</span><span class="n">Base</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Dags can originate from different places (user repos, master repo, ...)</span>
<span class="sd"> and also get executed in different places (different executors). This</span>
@@ -902,7 +951,7 @@
<span class="sd"> """</span>
<span class="nb">id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">,</span> <span class="n">primary_key</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">pickle</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">PickleType</span><span class="p">(</span><span class="n">pickler</span><span class="o">=</span><span class="n">dill</span><span class="p">))</span>
- <span class="n">created_dttm</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">DateTime</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="n">func</span><span class="o">.</span><span class="n">now</span><span class="p">())</span>
+ <span class="n">created_dttm</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">)</span>
<span class="n">pickle_hash</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Text</span><span class="p">)</span>
<span class="n">__tablename__</span> <span class="o">=</span> <span class="s2">"dag_pickle"</span>
@@ -912,7 +961,7 @@
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">dag</span><span class="p">,</span> <span class="s1">'template_env'</span><span class="p">):</span>
<span class="n">dag</span><span class="o">.</span><span class="n">template_env</span> <span class="o">=</span> <span class="kc">None</span>
<span class="bp">self</span><span class="o">.</span><span class="n">pickle_hash</span> <span class="o">=</span> <span class="nb">hash</span><span class="p">(</span><span class="n">dag</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">pickle</span> <span class="o">=</span> <span class="n">dag</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">pickle</span> <span class="o">=</span> <span class="n">dag</span></div>
<div class="viewcode-block" id="TaskInstance"><a class="viewcode-back" href="../../code.html#airflow.models.TaskInstance">[docs]</a><span class="k">class</span> <span class="nc">TaskInstance</span><span class="p">(</span><span class="n">Base</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
@@ -933,22 +982,23 @@
<span class="n">task_id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="n">ID_LEN</span><span class="p">),</span> <span class="n">primary_key</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">dag_id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="n">ID_LEN</span><span class="p">),</span> <span class="n">primary_key</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
- <span class="n">execution_date</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">DateTime</span><span class="p">,</span> <span class="n">primary_key</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
- <span class="n">start_date</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">DateTime</span><span class="p">)</span>
- <span class="n">end_date</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">DateTime</span><span class="p">)</span>
+ <span class="n">execution_date</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">,</span> <span class="n">primary_key</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="n">start_date</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">)</span>
+ <span class="n">end_date</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">)</span>
<span class="n">duration</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Float</span><span class="p">)</span>
<span class="n">state</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">20</span><span class="p">))</span>
<span class="n">_try_number</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="s1">'try_number'</span><span class="p">,</span> <span class="n">Integer</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
<span class="n">max_tries</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">)</span>
<span class="n">hostname</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">1000</span><span class="p">))</span>
<span class="n">unixname</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">1000</span><span class="p">))</span>
- <span class="n">job_id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">)</span>
+ <span class="n">job_id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">,</span> <span class="n">index</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">pool</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">50</span><span class="p">))</span>
<span class="n">queue</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">50</span><span class="p">))</span>
<span class="n">priority_weight</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">)</span>
<span class="n">operator</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">1000</span><span class="p">))</span>
- <span class="n">queued_dttm</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">DateTime</span><span class="p">)</span>
+ <span class="n">queued_dttm</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">)</span>
<span class="n">pid</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">)</span>
+ <span class="n">executor_config</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">PickleType</span><span class="p">(</span><span class="n">pickler</span><span class="o">=</span><span class="n">dill</span><span class="p">))</span>
<span class="n">__table_args__</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">Index</span><span class="p">(</span><span class="s1">'ti_dag_state'</span><span class="p">,</span> <span class="n">dag_id</span><span class="p">,</span> <span class="n">state</span><span class="p">),</span>
@@ -960,8 +1010,23 @@
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task</span><span class="p">,</span> <span class="n">execution_date</span><span class="p">,</span> <span class="n">state</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">dag_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_id</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">task_id</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">=</span> <span class="n">execution_date</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task</span> <span class="o">=</span> <span class="n">task</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_log</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">"airflow.task"</span><span class="p">)</span>
+
+ <span class="c1"># make sure we have a localized execution_date stored in UTC</span>
+ <span class="k">if</span> <span class="n">execution_date</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">timezone</span><span class="o">.</span><span class="n">is_localized</span><span class="p">(</span><span class="n">execution_date</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"execution date </span><span class="si">%s</span><span class="s2"> has no timezone information. Using "</span>
+ <span class="s2">"default from dag or system"</span><span class="p">,</span> <span class="n">execution_date</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">has_dag</span><span class="p">():</span>
+ <span class="n">execution_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">make_aware</span><span class="p">(</span><span class="n">execution_date</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">timezone</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">execution_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">make_aware</span><span class="p">(</span><span class="n">execution_date</span><span class="p">)</span>
+
+ <span class="n">execution_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">convert_to_utc</span><span class="p">(</span><span class="n">execution_date</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">=</span> <span class="n">execution_date</span>
+
<span class="bp">self</span><span class="o">.</span><span class="n">queue</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">queue</span>
<span class="bp">self</span><span class="o">.</span><span class="n">pool</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">pool</span>
<span class="bp">self</span><span class="o">.</span><span class="n">priority_weight</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">priority_weight_total</span>
@@ -972,8 +1037,11 @@
<span class="k">if</span> <span class="n">state</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">state</span>
<span class="bp">self</span><span class="o">.</span><span class="n">hostname</span> <span class="o">=</span> <span class="s1">''</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">executor_config</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">executor_config</span>
<span class="bp">self</span><span class="o">.</span><span class="n">init_on_load</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">_log</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">"airflow.task"</span><span class="p">)</span>
+ <span class="c1"># Is this TaskInstance being currently running within `airflow run --raw`.</span>
+ <span class="c1"># Not persisted to the database so only valid for the current process</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">raw</span> <span class="o">=</span> <span class="kc">False</span>
<div class="viewcode-block" id="TaskInstance.init_on_load"><a class="viewcode-back" href="../../code.html#airflow.models.TaskInstance.init_on_load">[docs]</a> <span class="nd">@reconstructor</span>
<span class="k">def</span> <span class="nf">init_on_load</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
@@ -1127,6 +1195,8 @@
<span class="sd"> :param job_id: job ID (needs more details)</span>
<span class="sd"> :param pool: the Airflow pool that the task should run in</span>
<span class="sd"> :type pool: unicode</span>
+<span class="sd"> :param cfg_path: the Path to the configuration file</span>
+<span class="sd"> :type cfg_path: basestring</span>
<span class="sd"> :return: shell command that can be used to run the task instance</span>
<span class="sd"> """</span>
<span class="n">iso</span> <span class="o">=</span> <span class="n">execution_date</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
@@ -1148,34 +1218,51 @@
<span class="nd">@property</span>
<span class="k">def</span> <span class="nf">log_filepath</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">iso</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
- <span class="n">log</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">expanduser</span><span class="p">(</span><span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'BASE_LOG_FOLDER'</span><span class="p">))</span>
+ <span class="n">log</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">expanduser</span><span class="p">(</span><span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'BASE_LOG_FOLDER'</span><span class="p">))</span>
<span class="k">return</span> <span class="p">(</span>
<span class="s2">"</span><span class="si">{log}</span><span class="s2">/</span><span class="si">{self.dag_id}</span><span class="s2">/</span><span class="si">{self.task_id}</span><span class="s2">/</span><span class="si">{iso}</span><span class="s2">.log"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
<span class="nd">@property</span>
<span class="k">def</span> <span class="nf">log_url</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="n">iso</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
- <span class="n">BASE_URL</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'webserver'</span><span class="p">,</span> <span class="s1">'BASE_URL'</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">BASE_URL</span> <span class="o">+</span> <span class="p">(</span>
- <span class="s2">"/admin/airflow/log"</span>
- <span class="s2">"?dag_id=</span><span class="si">{self.dag_id}</span><span class="s2">"</span>
- <span class="s2">"&task_id=</span><span class="si">{self.task_id}</span><span class="s2">"</span>
- <span class="s2">"&execution_date=</span><span class="si">{iso}</span><span class="s2">"</span>
- <span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
+ <span class="n">iso</span> <span class="o">=</span> <span class="n">quote</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="o">.</span><span class="n">isoformat</span><span class="p">())</span>
+ <span class="n">BASE_URL</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'webserver'</span><span class="p">,</span> <span class="s1">'BASE_URL'</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">settings</span><span class="o">.</span><span class="n">RBAC</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">BASE_URL</span> <span class="o">+</span> <span class="p">(</span>
+ <span class="s2">"/log/list/"</span>
+ <span class="s2">"?_flt_3_dag_id=</span><span class="si">{self.dag_id}</span><span class="s2">"</span>
+ <span class="s2">"&_flt_3_task_id=</span><span class="si">{self.task_id}</span><span class="s2">"</span>
+ <span class="s2">"&_flt_3_execution_date=</span><span class="si">{iso}</span><span class="s2">"</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">BASE_URL</span> <span class="o">+</span> <span class="p">(</span>
+ <span class="s2">"/admin/airflow/log"</span>
+ <span class="s2">"?dag_id=</span><span class="si">{self.dag_id}</span><span class="s2">"</span>
+ <span class="s2">"&task_id=</span><span class="si">{self.task_id}</span><span class="s2">"</span>
+ <span class="s2">"&execution_date=</span><span class="si">{iso}</span><span class="s2">"</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
<span class="nd">@property</span>
<span class="k">def</span> <span class="nf">mark_success_url</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="n">iso</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
- <span class="n">BASE_URL</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'webserver'</span><span class="p">,</span> <span class="s1">'BASE_URL'</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">BASE_URL</span> <span class="o">+</span> <span class="p">(</span>
- <span class="s2">"/admin/airflow/action"</span>
- <span class="s2">"?action=success"</span>
- <span class="s2">"&task_id=</span><span class="si">{self.task_id}</span><span class="s2">"</span>
- <span class="s2">"&dag_id=</span><span class="si">{self.dag_id}</span><span class="s2">"</span>
- <span class="s2">"&execution_date=</span><span class="si">{iso}</span><span class="s2">"</span>
- <span class="s2">"&upstream=false"</span>
- <span class="s2">"&downstream=false"</span>
- <span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
+ <span class="n">iso</span> <span class="o">=</span> <span class="n">quote</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="o">.</span><span class="n">isoformat</span><span class="p">())</span>
+ <span class="n">BASE_URL</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'webserver'</span><span class="p">,</span> <span class="s1">'BASE_URL'</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">settings</span><span class="o">.</span><span class="n">RBAC</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">BASE_URL</span> <span class="o">+</span> <span class="p">(</span>
+ <span class="s2">"/success"</span>
+ <span class="s2">"?task_id=</span><span class="si">{self.task_id}</span><span class="s2">"</span>
+ <span class="s2">"&dag_id=</span><span class="si">{self.dag_id}</span><span class="s2">"</span>
+ <span class="s2">"&execution_date=</span><span class="si">{iso}</span><span class="s2">"</span>
+ <span class="s2">"&upstream=false"</span>
+ <span class="s2">"&downstream=false"</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">BASE_URL</span> <span class="o">+</span> <span class="p">(</span>
+ <span class="s2">"/admin/airflow/success"</span>
+ <span class="s2">"?task_id=</span><span class="si">{self.task_id}</span><span class="s2">"</span>
+ <span class="s2">"&dag_id=</span><span class="si">{self.dag_id}</span><span class="s2">"</span>
+ <span class="s2">"&execution_date=</span><span class="si">{iso}</span><span class="s2">"</span>
+ <span class="s2">"&upstream=false"</span>
+ <span class="s2">"&downstream=false"</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
<div class="viewcode-block" id="TaskInstance.current_state"><a class="viewcode-back" href="../../code.html#airflow.models.TaskInstance.current_state">[docs]</a> <span class="nd">@provide_session</span>
<span class="k">def</span> <span class="nf">current_state</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
@@ -1236,6 +1323,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">max_tries</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">max_tries</span>
<span class="bp">self</span><span class="o">.</span><span class="n">hostname</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">hostname</span>
<span class="bp">self</span><span class="o">.</span><span class="n">pid</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">pid</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">executor_config</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">executor_config</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="kc">None</span></div>
@@ -1258,10 +1346,11 @@
<span class="sd"> """</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span>
- <span class="k">def</span> <span class="nf">set_state</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">session</span><span class="p">):</span>
+ <span class="nd">@provide_session</span>
+ <span class="k">def</span> <span class="nf">set_state</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">state</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">start_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">start_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
<TRUNCATED>
[28/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/mlengine_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/mlengine_operator.html b/_modules/airflow/contrib/operators/mlengine_operator.html
index b322476..c7743c2 100644
--- a/_modules/airflow/contrib/operators/mlengine_operator.html
+++ b/_modules/airflow/contrib/operators/mlengine_operator.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -184,77 +186,17 @@
<span class="c1"># limitations under the License.</span>
<span class="kn">import</span> <span class="nn">re</span>
-<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">settings</span>
+<span class="kn">from</span> <span class="nn">apiclient</span> <span class="k">import</span> <span class="n">errors</span>
+
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_mlengine_hook</span> <span class="k">import</span> <span class="n">MLEngineHook</span>
<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
<span class="kn">from</span> <span class="nn">airflow.operators</span> <span class="k">import</span> <span class="n">BaseOperator</span>
<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
-<span class="kn">from</span> <span class="nn">apiclient</span> <span class="k">import</span> <span class="n">errors</span>
-
<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
<span class="n">log</span> <span class="o">=</span> <span class="n">LoggingMixin</span><span class="p">()</span><span class="o">.</span><span class="n">log</span>
-<span class="k">def</span> <span class="nf">_create_prediction_input</span><span class="p">(</span><span class="n">project_id</span><span class="p">,</span>
- <span class="n">region</span><span class="p">,</span>
- <span class="n">data_format</span><span class="p">,</span>
- <span class="n">input_paths</span><span class="p">,</span>
- <span class="n">output_path</span><span class="p">,</span>
- <span class="n">model_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">version_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">uri</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">max_worker_count</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">runtime_version</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Create the batch prediction input from the given parameters.</span>
-
-<span class="sd"> Args:</span>
-<span class="sd"> A subset of arguments documented in __init__ method of class</span>
-<span class="sd"> MLEngineBatchPredictionOperator</span>
-
-<span class="sd"> Returns:</span>
-<span class="sd"> A dictionary representing the predictionInput object as documented</span>
-<span class="sd"> in https://cloud.google.com/ml-engine/reference/rest/v1/projects.jobs.</span>
-
-<span class="sd"> Raises:</span>
-<span class="sd"> ValueError: if a unique model/version origin cannot be determined.</span>
-<span class="sd"> """</span>
- <span class="n">prediction_input</span> <span class="o">=</span> <span class="p">{</span>
- <span class="s1">'dataFormat'</span><span class="p">:</span> <span class="n">data_format</span><span class="p">,</span>
- <span class="s1">'inputPaths'</span><span class="p">:</span> <span class="n">input_paths</span><span class="p">,</span>
- <span class="s1">'outputPath'</span><span class="p">:</span> <span class="n">output_path</span><span class="p">,</span>
- <span class="s1">'region'</span><span class="p">:</span> <span class="n">region</span>
- <span class="p">}</span>
-
- <span class="k">if</span> <span class="n">uri</span><span class="p">:</span>
- <span class="k">if</span> <span class="n">model_name</span> <span class="ow">or</span> <span class="n">version_name</span><span class="p">:</span>
- <span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
- <span class="s1">'Ambiguous model origin: Both uri and model/version name are provided.'</span>
- <span class="p">)</span>
- <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'Ambiguous model origin.'</span><span class="p">)</span>
- <span class="n">prediction_input</span><span class="p">[</span><span class="s1">'uri'</span><span class="p">]</span> <span class="o">=</span> <span class="n">uri</span>
- <span class="k">elif</span> <span class="n">model_name</span><span class="p">:</span>
- <span class="n">origin_name</span> <span class="o">=</span> <span class="s1">'projects/</span><span class="si">{}</span><span class="s1">/models/</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">project_id</span><span class="p">,</span> <span class="n">model_name</span><span class="p">)</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">version_name</span><span class="p">:</span>
- <span class="n">prediction_input</span><span class="p">[</span><span class="s1">'modelName'</span><span class="p">]</span> <span class="o">=</span> <span class="n">origin_name</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">prediction_input</span><span class="p">[</span><span class="s1">'versionName'</span><span class="p">]</span> <span class="o">=</span> \
- <span class="n">origin_name</span> <span class="o">+</span> <span class="s1">'/versions/</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">version_name</span><span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
- <span class="s1">'Missing model origin: Batch prediction expects a model, '</span>
- <span class="s1">'a model & version combination, or a URI to savedModel.'</span><span class="p">)</span>
- <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'Missing model origin.'</span><span class="p">)</span>
-
- <span class="k">if</span> <span class="n">max_worker_count</span><span class="p">:</span>
- <span class="n">prediction_input</span><span class="p">[</span><span class="s1">'maxWorkerCount'</span><span class="p">]</span> <span class="o">=</span> <span class="n">max_worker_count</span>
- <span class="k">if</span> <span class="n">runtime_version</span><span class="p">:</span>
- <span class="n">prediction_input</span><span class="p">[</span><span class="s1">'runtimeVersion'</span><span class="p">]</span> <span class="o">=</span> <span class="n">runtime_version</span>
-
- <span class="k">return</span> <span class="n">prediction_input</span>
-
-
<span class="k">def</span> <span class="nf">_normalize_mlengine_job_id</span><span class="p">(</span><span class="n">job_id</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Replaces invalid MLEngine job_id characters with '_'.</span>
@@ -268,10 +210,27 @@
<span class="sd"> Returns:</span>
<span class="sd"> A valid job_id representation.</span>
<span class="sd"> """</span>
- <span class="n">match</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="sa">r</span><span class="s1">'\d'</span><span class="p">,</span> <span class="n">job_id</span><span class="p">)</span>
+
+ <span class="c1"># Add a prefix when a job_id starts with a digit or a template</span>
+ <span class="n">match</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="sa">r</span><span class="s1">'\d|\{{2}'</span><span class="p">,</span> <span class="n">job_id</span><span class="p">)</span>
<span class="k">if</span> <span class="n">match</span> <span class="ow">and</span> <span class="n">match</span><span class="o">.</span><span class="n">start</span><span class="p">()</span> <span class="ow">is</span> <span class="mi">0</span><span class="p">:</span>
- <span class="n">job_id</span> <span class="o">=</span> <span class="s1">'z_</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">job_id</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="s1">'[^0-9a-zA-Z]+'</span><span class="p">,</span> <span class="s1">'_'</span><span class="p">,</span> <span class="n">job_id</span><span class="p">)</span>
+ <span class="n">job</span> <span class="o">=</span> <span class="s1">'z_</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">job_id</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">job</span> <span class="o">=</span> <span class="n">job_id</span>
+
+ <span class="c1"># Clean up 'bad' characters except templates</span>
+ <span class="n">tracker</span> <span class="o">=</span> <span class="mi">0</span>
+ <span class="n">cleansed_job_id</span> <span class="o">=</span> <span class="s1">''</span>
+ <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="n">re</span><span class="o">.</span><span class="n">finditer</span><span class="p">(</span><span class="sa">r</span><span class="s1">'\{{2}.+?\}</span><span class="si">{2}</span><span class="s1">'</span><span class="p">,</span> <span class="n">job</span><span class="p">):</span>
+ <span class="n">cleansed_job_id</span> <span class="o">+=</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="sa">r</span><span class="s1">'[^0-9a-zA-Z]+'</span><span class="p">,</span> <span class="s1">'_'</span><span class="p">,</span>
+ <span class="n">job</span><span class="p">[</span><span class="n">tracker</span><span class="p">:</span><span class="n">m</span><span class="o">.</span><span class="n">start</span><span class="p">()])</span>
+ <span class="n">cleansed_job_id</span> <span class="o">+=</span> <span class="n">job</span><span class="p">[</span><span class="n">m</span><span class="o">.</span><span class="n">start</span><span class="p">():</span><span class="n">m</span><span class="o">.</span><span class="n">end</span><span class="p">()]</span>
+ <span class="n">tracker</span> <span class="o">=</span> <span class="n">m</span><span class="o">.</span><span class="n">end</span><span class="p">()</span>
+
+ <span class="c1"># Clean up last substring or the full string if no templates</span>
+ <span class="n">cleansed_job_id</span> <span class="o">+=</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="sa">r</span><span class="s1">'[^0-9a-zA-Z]+'</span><span class="p">,</span> <span class="s1">'_'</span><span class="p">,</span> <span class="n">job</span><span class="p">[</span><span class="n">tracker</span><span class="p">:])</span>
+
+ <span class="k">return</span> <span class="n">cleansed_job_id</span>
<div class="viewcode-block" id="MLEngineBatchPredictionOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.mlengine_operator.MLEngineBatchPredictionOperator">[docs]</a><span class="k">class</span> <span class="nc">MLEngineBatchPredictionOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
@@ -289,14 +248,20 @@
<span class="sd"> In options 2 and 3, both model and version name should contain the</span>
<span class="sd"> minimal identifier. For instance, call</span>
+
+<span class="sd"> ::</span>
+
<span class="sd"> MLEngineBatchPredictionOperator(</span>
<span class="sd"> ...,</span>
<span class="sd"> model_name='my_model',</span>
<span class="sd"> version_name='my_version',</span>
<span class="sd"> ...)</span>
+
<span class="sd"> if the desired model version is</span>
<span class="sd"> "projects/my_project/models/my_model/versions/my_version".</span>
+<span class="sd"> See https://cloud.google.com/ml-engine/reference/rest/v1/projects.jobs</span>
+<span class="sd"> for further documentation on the parameters.</span>
<span class="sd"> :param project_id: The Google Cloud project name where the</span>
<span class="sd"> prediction job is submitted.</span>
@@ -358,11 +323,18 @@
<span class="sd"> :type delegate_to: string</span>
<span class="sd"> Raises:</span>
-<span class="sd"> ValueError: if a unique model/version origin cannot be determined.</span>
+<span class="sd"> ``ValueError``: if a unique model/version origin cannot be determined.</span>
<span class="sd"> """</span>
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span>
- <span class="s2">"prediction_job_request"</span><span class="p">,</span>
+ <span class="s1">'_project_id'</span><span class="p">,</span>
+ <span class="s1">'_job_id'</span><span class="p">,</span>
+ <span class="s1">'_region'</span><span class="p">,</span>
+ <span class="s1">'_input_paths'</span><span class="p">,</span>
+ <span class="s1">'_output_path'</span><span class="p">,</span>
+ <span class="s1">'_model_name'</span><span class="p">,</span>
+ <span class="s1">'_version_name'</span><span class="p">,</span>
+ <span class="s1">'_uri'</span><span class="p">,</span>
<span class="p">]</span>
<span class="nd">@apply_defaults</span>
@@ -384,45 +356,91 @@
<span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">MLEngineBatchPredictionOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_project_id</span> <span class="o">=</span> <span class="n">project_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_job_id</span> <span class="o">=</span> <span class="n">job_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_region</span> <span class="o">=</span> <span class="n">region</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_data_format</span> <span class="o">=</span> <span class="n">data_format</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_input_paths</span> <span class="o">=</span> <span class="n">input_paths</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_output_path</span> <span class="o">=</span> <span class="n">output_path</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_model_name</span> <span class="o">=</span> <span class="n">model_name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_version_name</span> <span class="o">=</span> <span class="n">version_name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_uri</span> <span class="o">=</span> <span class="n">uri</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_max_worker_count</span> <span class="o">=</span> <span class="n">max_worker_count</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_runtime_version</span> <span class="o">=</span> <span class="n">runtime_version</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_project_id</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Google Cloud project id is required.'</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job_id</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'An unique job id is required for Google MLEngine prediction '</span>
+ <span class="s1">'job.'</span><span class="p">)</span>
- <span class="k">try</span><span class="p">:</span>
- <span class="n">prediction_input</span> <span class="o">=</span> <span class="n">_create_prediction_input</span><span class="p">(</span>
- <span class="n">project_id</span><span class="p">,</span> <span class="n">region</span><span class="p">,</span> <span class="n">data_format</span><span class="p">,</span> <span class="n">input_paths</span><span class="p">,</span> <span class="n">output_path</span><span class="p">,</span>
- <span class="n">model_name</span><span class="p">,</span> <span class="n">version_name</span><span class="p">,</span> <span class="n">uri</span><span class="p">,</span> <span class="n">max_worker_count</span><span class="p">,</span>
- <span class="n">runtime_version</span><span class="p">)</span>
- <span class="k">except</span> <span class="ne">ValueError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
- <span class="s1">'Cannot create batch prediction job request due to: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
- <span class="n">e</span>
- <span class="p">)</span>
- <span class="k">raise</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_uri</span><span class="p">:</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_model_name</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">_version_name</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Ambiguous model origin: Both uri and '</span>
+ <span class="s1">'model/version name are provided.'</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">prediction_job_request</span> <span class="o">=</span> <span class="p">{</span>
- <span class="s1">'jobId'</span><span class="p">:</span> <span class="n">_normalize_mlengine_job_id</span><span class="p">(</span><span class="n">job_id</span><span class="p">),</span>
- <span class="s1">'predictionInput'</span><span class="p">:</span> <span class="n">prediction_input</span>
- <span class="p">}</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_version_name</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_model_name</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'Missing model: Batch prediction expects '</span>
+ <span class="s1">'a model name when a version name is provided.'</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_uri</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">_model_name</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'Missing model origin: Batch prediction expects a model, '</span>
+ <span class="s1">'a model & version combination, or a URI to a savedModel.'</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
- <span class="n">hook</span> <span class="o">=</span> <span class="n">MLEngineHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+ <span class="n">job_id</span> <span class="o">=</span> <span class="n">_normalize_mlengine_job_id</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_job_id</span><span class="p">)</span>
+ <span class="n">prediction_request</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'jobId'</span><span class="p">:</span> <span class="n">job_id</span><span class="p">,</span>
+ <span class="s1">'predictionInput'</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s1">'dataFormat'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_data_format</span><span class="p">,</span>
+ <span class="s1">'inputPaths'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_input_paths</span><span class="p">,</span>
+ <span class="s1">'outputPath'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_output_path</span><span class="p">,</span>
+ <span class="s1">'region'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_region</span>
+ <span class="p">}</span>
+ <span class="p">}</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_uri</span><span class="p">:</span>
+ <span class="n">prediction_request</span><span class="p">[</span><span class="s1">'predictionInput'</span><span class="p">][</span><span class="s1">'uri'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_uri</span>
+ <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_model_name</span><span class="p">:</span>
+ <span class="n">origin_name</span> <span class="o">=</span> <span class="s1">'projects/</span><span class="si">{}</span><span class="s1">/models/</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_model_name</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_version_name</span><span class="p">:</span>
+ <span class="n">prediction_request</span><span class="p">[</span><span class="s1">'predictionInput'</span><span class="p">][</span>
+ <span class="s1">'modelName'</span><span class="p">]</span> <span class="o">=</span> <span class="n">origin_name</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">prediction_request</span><span class="p">[</span><span class="s1">'predictionInput'</span><span class="p">][</span><span class="s1">'versionName'</span><span class="p">]</span> <span class="o">=</span> \
+ <span class="n">origin_name</span> <span class="o">+</span> <span class="s1">'/versions/</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_version_name</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_max_worker_count</span><span class="p">:</span>
+ <span class="n">prediction_request</span><span class="p">[</span><span class="s1">'predictionInput'</span><span class="p">][</span>
+ <span class="s1">'maxWorkerCount'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_max_worker_count</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_runtime_version</span><span class="p">:</span>
+ <span class="n">prediction_request</span><span class="p">[</span><span class="s1">'predictionInput'</span><span class="p">][</span>
+ <span class="s1">'runtimeVersion'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_runtime_version</span>
+
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">MLEngineHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_gcp_conn_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_delegate_to</span><span class="p">)</span>
+
+ <span class="c1"># Helper method to check if the existing job's prediction input is the</span>
+ <span class="c1"># same as the request we get here.</span>
<span class="k">def</span> <span class="nf">check_existing_job</span><span class="p">(</span><span class="n">existing_job</span><span class="p">):</span>
<span class="k">return</span> <span class="n">existing_job</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'predictionInput'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="o">==</span> \
- <span class="bp">self</span><span class="o">.</span><span class="n">prediction_job_request</span><span class="p">[</span><span class="s1">'predictionInput'</span><span class="p">]</span>
+ <span class="n">prediction_request</span><span class="p">[</span><span class="s1">'predictionInput'</span><span class="p">]</span>
+
<span class="k">try</span><span class="p">:</span>
<span class="n">finished_prediction_job</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">create_job</span><span class="p">(</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">prediction_job_request</span><span class="p">,</span>
- <span class="n">check_existing_job</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_project_id</span><span class="p">,</span> <span class="n">prediction_request</span><span class="p">,</span> <span class="n">check_existing_job</span><span class="p">)</span>
<span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span><span class="p">:</span>
<span class="k">raise</span>
<span class="k">if</span> <span class="n">finished_prediction_job</span><span class="p">[</span><span class="s1">'state'</span><span class="p">]</span> <span class="o">!=</span> <span class="s1">'SUCCEEDED'</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
- <span class="s1">'Batch prediction job failed: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
- <span class="nb">str</span><span class="p">(</span><span class="n">finished_prediction_job</span><span class="p">))</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">'MLEngine batch prediction job failed: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="nb">str</span><span class="p">(</span><span class="n">finished_prediction_job</span><span class="p">)))</span>
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="n">finished_prediction_job</span><span class="p">[</span><span class="s1">'errorMessage'</span><span class="p">])</span>
<span class="k">return</span> <span class="n">finished_prediction_job</span><span class="p">[</span><span class="s1">'predictionOutput'</span><span class="p">]</span></div>
@@ -445,8 +463,9 @@
<span class="sd"> :type model: dict</span>
<span class="sd"> :param operation: The operation to perform. Available operations are:</span>
-<span class="sd"> 'create': Creates a new model as provided by the `model` parameter.</span>
-<span class="sd"> 'get': Gets a particular model where the name is specified in `model`.</span>
+
+<span class="sd"> * ``create``: Creates a new model as provided by the `model` parameter.</span>
+<span class="sd"> * ``get``: Gets a particular model where the name is specified in `model`.</span>
<span class="sd"> :param gcp_conn_id: The connection ID to use when fetching connection info.</span>
<span class="sd"> :type gcp_conn_id: string</span>
@@ -515,23 +534,25 @@
<span class="sd"> :type version: dict</span>
<span class="sd"> :param operation: The operation to perform. Available operations are:</span>
-<span class="sd"> 'create': Creates a new version in the model specified by `model_name`,</span>
+
+<span class="sd"> * ``create``: Creates a new version in the model specified by `model_name`,</span>
<span class="sd"> in which case the `version` parameter should contain all the</span>
<span class="sd"> information to create that version</span>
<span class="sd"> (e.g. `name`, `deploymentUrl`).</span>
-<span class="sd"> 'get': Gets full information of a particular version in the model</span>
+
+<span class="sd"> * ``get``: Gets full information of a particular version in the model</span>
<span class="sd"> specified by `model_name`.</span>
<span class="sd"> The name of the version should be specified in the `version`</span>
<span class="sd"> parameter.</span>
-<span class="sd"> 'list': Lists all available versions of the model specified</span>
+<span class="sd"> * ``list``: Lists all available versions of the model specified</span>
<span class="sd"> by `model_name`.</span>
-<span class="sd"> 'delete': Deletes the version specified in `version` parameter from the</span>
+<span class="sd"> * ``delete``: Deletes the version specified in `version` parameter from the</span>
<span class="sd"> model specified by `model_name`).</span>
<span class="sd"> The name of the version should be specified in the `version`</span>
<span class="sd"> parameter.</span>
-<span class="sd"> :type operation: string</span>
+<span class="sd"> :type operation: string</span>
<span class="sd"> :param gcp_conn_id: The connection ID to use when fetching connection info.</span>
<span class="sd"> :type gcp_conn_id: string</span>
@@ -581,9 +602,8 @@
<span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">create_version</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_model_name</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_version</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_operation</span> <span class="o">==</span> <span class="s1">'set_default'</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">set_default_version</span><span class="p">(</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">_project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_model_name</span><span class="p">,</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">_version</span><span class="p">[</span><span class="s1">'name'</span><span class="p">])</span>
+ <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">set_default_version</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_model_name</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_version</span><span class="p">[</span><span class="s1">'name'</span><span class="p">])</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_operation</span> <span class="o">==</span> <span class="s1">'list'</span><span class="p">:</span>
<span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">list_versions</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_model_name</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_operation</span> <span class="o">==</span> <span class="s1">'delete'</span><span class="p">:</span>
@@ -625,6 +645,16 @@
<span class="sd"> :param scale_tier: Resource tier for MLEngine training job.</span>
<span class="sd"> :type scale_tier: string</span>
+<span class="sd"> :param runtime_version: The Google Cloud ML runtime version to use for training.</span>
+<span class="sd"> :type runtime_version: string</span>
+
+<span class="sd"> :param python_version: The version of Python used in training.</span>
+<span class="sd"> :type python_version: string</span>
+
+<span class="sd"> :param job_dir: A Google Cloud Storage path in which to store training</span>
+<span class="sd"> outputs and other data needed for training.</span>
+<span class="sd"> :type job_dir: string</span>
+
<span class="sd"> :param gcp_conn_id: The connection ID to use when fetching connection info.</span>
<span class="sd"> :type gcp_conn_id: string</span>
@@ -648,6 +678,9 @@
<span class="s1">'_training_args'</span><span class="p">,</span>
<span class="s1">'_region'</span><span class="p">,</span>
<span class="s1">'_scale_tier'</span><span class="p">,</span>
+ <span class="s1">'_runtime_version'</span><span class="p">,</span>
+ <span class="s1">'_python_version'</span><span class="p">,</span>
+ <span class="s1">'_job_dir'</span>
<span class="p">]</span>
<span class="nd">@apply_defaults</span>
@@ -659,6 +692,9 @@
<span class="n">training_args</span><span class="p">,</span>
<span class="n">region</span><span class="p">,</span>
<span class="n">scale_tier</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">runtime_version</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">python_version</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">job_dir</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">mode</span><span class="o">=</span><span class="s1">'PRODUCTION'</span><span class="p">,</span>
@@ -672,6 +708,9 @@
<span class="bp">self</span><span class="o">.</span><span class="n">_training_args</span> <span class="o">=</span> <span class="n">training_args</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_region</span> <span class="o">=</span> <span class="n">region</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_scale_tier</span> <span class="o">=</span> <span class="n">scale_tier</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_runtime_version</span> <span class="o">=</span> <span class="n">runtime_version</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_python_version</span> <span class="o">=</span> <span class="n">python_version</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_job_dir</span> <span class="o">=</span> <span class="n">job_dir</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_mode</span> <span class="o">=</span> <span class="n">mode</span>
@@ -706,9 +745,19 @@
<span class="p">}</span>
<span class="p">}</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_runtime_version</span><span class="p">:</span>
+ <span class="n">training_request</span><span class="p">[</span><span class="s1">'trainingInput'</span><span class="p">][</span><span class="s1">'runtimeVersion'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_runtime_version</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_python_version</span><span class="p">:</span>
+ <span class="n">training_request</span><span class="p">[</span><span class="s1">'trainingInput'</span><span class="p">][</span><span class="s1">'pythonVersion'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_python_version</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job_dir</span><span class="p">:</span>
+ <span class="n">training_request</span><span class="p">[</span><span class="s1">'trainingInput'</span><span class="p">][</span><span class="s1">'jobDir'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job_dir</span>
+
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mode</span> <span class="o">==</span> <span class="s1">'DRY_RUN'</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'In dry_run mode.'</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'MLEngine Training job request is: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">training_request</span><span class="p">))</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'MLEngine Training job request is: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">training_request</span><span class="p">))</span>
<span class="k">return</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">MLEngineHook</span><span class="p">(</span>
@@ -719,6 +768,7 @@
<span class="k">def</span> <span class="nf">check_existing_job</span><span class="p">(</span><span class="n">existing_job</span><span class="p">):</span>
<span class="k">return</span> <span class="n">existing_job</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'trainingInput'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="o">==</span> \
<span class="n">training_request</span><span class="p">[</span><span class="s1">'trainingInput'</span><span class="p">]</span>
+
<span class="k">try</span><span class="p">:</span>
<span class="n">finished_training_job</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">create_job</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_project_id</span><span class="p">,</span> <span class="n">training_request</span><span class="p">,</span> <span class="n">check_existing_job</span><span class="p">)</span>
[18/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/executors/local_executor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/executors/local_executor.html b/_modules/airflow/executors/local_executor.html
index cfbf241..24436b4 100644
--- a/_modules/airflow/executors/local_executor.html
+++ b/_modules/airflow/executors/local_executor.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,49 @@
<h1>Source code for airflow.executors.local_executor</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="sd">"""</span>
+<span class="sd">LocalExecutor runs tasks by spawning processes in a controlled fashion in different</span>
+<span class="sd">modes. Given that BaseExecutor has the option to receive a `parallelism` parameter to</span>
+<span class="sd">limit the number of process spawned, when this parameter is `0` the number of processes</span>
+<span class="sd">that LocalExecutor can spawn is unlimited.</span>
+
+<span class="sd">The following strategies are implemented:</span>
+<span class="sd">1. Unlimited Parallelism (self.parallelism == 0): In this strategy, LocalExecutor will</span>
+<span class="sd">spawn a process every time `execute_async` is called, that is, every task submitted to the</span>
+<span class="sd">LocalExecutor will be executed in its own process. Once the task is executed and the</span>
+<span class="sd">result stored in the `result_queue`, the process terminates. There is no need for a</span>
+<span class="sd">`task_queue` in this approach, since as soon as a task is received a new process will be</span>
+<span class="sd">allocated to the task. Processes used in this strategy are of class LocalWorker.</span>
+
+<span class="sd">2. Limited Parallelism (self.parallelism > 0): In this strategy, the LocalExecutor spawns</span>
+<span class="sd">the number of processes equal to the value of `self.parallelism` at `start` time,</span>
+<span class="sd">using a `task_queue` to coordinate the ingestion of tasks and the work distribution among</span>
+<span class="sd">the workers, which will take a task as soon as they are ready. During the lifecycle of</span>
+<span class="sd">the LocalExecutor, the worker processes are running waiting for tasks, once the</span>
+<span class="sd">LocalExecutor receives the call to shutdown the executor a poison token is sent to the</span>
+<span class="sd">workers to terminate them. Processes used in this strategy are of class QueuedLocalWorker.</span>
+
+<span class="sd">Arguably, `SequentialExecutor` could be thought as a LocalExecutor with limited</span>
+<span class="sd">parallelism of just 1 worker, i.e. `self.parallelism = 1`.</span>
+<span class="sd">This option could lead to the unification of the executor implementations, running</span>
+<span class="sd">locally, into just one `LocalExecutor` with multiple modes.</span>
+<span class="sd">"""</span>
<span class="kn">import</span> <span class="nn">multiprocessing</span>
<span class="kn">import</span> <span class="nn">subprocess</span>
@@ -187,20 +221,63 @@
<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">range</span>
-<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">configuration</span>
<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="k">import</span> <span class="n">BaseExecutor</span>
<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span>
-<span class="n">PARALLELISM</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'PARALLELISM'</span><span class="p">)</span>
-
<span class="k">class</span> <span class="nc">LocalWorker</span><span class="p">(</span><span class="n">multiprocessing</span><span class="o">.</span><span class="n">Process</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+
+ <span class="sd">"""LocalWorker Process implementation to run airflow commands. Executes the given</span>
+<span class="sd"> command and puts the result into a result queue when done, terminating execution."""</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result_queue</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> :param result_queue: the queue to store result states tuples (key, State)</span>
+<span class="sd"> :type result_queue: multiprocessing.Queue</span>
+<span class="sd"> """</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">LocalWorker</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">daemon</span> <span class="o">=</span> <span class="kc">True</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="n">result_queue</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">key</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">command</span> <span class="o">=</span> <span class="kc">None</span>
+
+ <span class="k">def</span> <span class="nf">execute_work</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Executes command received and stores result state in queue.</span>
+<span class="sd"> :param key: the key to identify the TI</span>
+<span class="sd"> :type key: Tuple(dag_id, task_id, execution_date)</span>
+<span class="sd"> :param command: the command to execute</span>
+<span class="sd"> :type command: string</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">key</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">return</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"</span><span class="si">%s</span><span class="s2"> running </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="p">,</span> <span class="n">command</span><span class="p">)</span>
+ <span class="n">command</span> <span class="o">=</span> <span class="s2">"exec bash -c '</span><span class="si">{0}</span><span class="s2">'"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">command</span><span class="p">)</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">subprocess</span><span class="o">.</span><span class="n">check_call</span><span class="p">(</span><span class="n">command</span><span class="p">,</span> <span class="n">shell</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">close_fds</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span>
+ <span class="k">except</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">CalledProcessError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">FAILED</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Failed to execute task </span><span class="si">%s</span><span class="s2">."</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
+ <span class="c1"># TODO: Why is this commented out?</span>
+ <span class="c1"># raise e</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">state</span><span class="p">))</span>
+
+ <span class="k">def</span> <span class="nf">run</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">execute_work</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">key</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">command</span><span class="p">)</span>
+ <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">QueuedLocalWorker</span><span class="p">(</span><span class="n">LocalWorker</span><span class="p">):</span>
+
+ <span class="sd">"""LocalWorker implementation that is waiting for tasks from a queue and will</span>
+<span class="sd"> continue executing commands as they become available in the queue. It will terminate</span>
+<span class="sd"> execution once the poison token is found."""</span>
+
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_queue</span><span class="p">,</span> <span class="n">result_queue</span><span class="p">):</span>
- <span class="n">multiprocessing</span><span class="o">.</span><span class="n">Process</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">QueuedLocalWorker</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">result_queue</span><span class="o">=</span><span class="n">result_queue</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span> <span class="o">=</span> <span class="n">task_queue</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="n">result_queue</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">daemon</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">def</span> <span class="nf">run</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
@@ -209,55 +286,120 @@
<span class="c1"># Received poison pill, no more tasks to run</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">task_done</span><span class="p">()</span>
<span class="k">break</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"</span><span class="si">%s</span><span class="s2"> running </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="p">,</span> <span class="n">command</span><span class="p">)</span>
- <span class="n">command</span> <span class="o">=</span> <span class="s2">"exec bash -c '</span><span class="si">{0}</span><span class="s2">'"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">command</span><span class="p">)</span>
- <span class="k">try</span><span class="p">:</span>
- <span class="n">subprocess</span><span class="o">.</span><span class="n">check_call</span><span class="p">(</span><span class="n">command</span><span class="p">,</span> <span class="n">shell</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
- <span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span>
- <span class="k">except</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">CalledProcessError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
- <span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">FAILED</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Failed to execute task </span><span class="si">%s</span><span class="s2">."</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
- <span class="c1"># TODO: Why is this commented out?</span>
- <span class="c1"># raise e</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">state</span><span class="p">))</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">execute_work</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">task_done</span><span class="p">()</span>
<span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
-<div class="viewcode-block" id="LocalExecutor"><a class="viewcode-back" href="../../../code.html#airflow.executors.LocalExecutor">[docs]</a><span class="k">class</span> <span class="nc">LocalExecutor</span><span class="p">(</span><span class="n">BaseExecutor</span><span class="p">):</span>
+<div class="viewcode-block" id="LocalExecutor"><a class="viewcode-back" href="../../../code.html#airflow.executors.local_executor.LocalExecutor">[docs]</a><span class="k">class</span> <span class="nc">LocalExecutor</span><span class="p">(</span><span class="n">BaseExecutor</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> LocalExecutor executes tasks locally in parallel. It uses the</span>
<span class="sd"> multiprocessing Python library and queues to parallelize the execution</span>
<span class="sd"> of tasks.</span>
<span class="sd"> """</span>
+ <span class="k">class</span> <span class="nc">_UnlimitedParallelism</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+ <span class="sd">"""Implements LocalExecutor with unlimited parallelism, starting one process</span>
+<span class="sd"> per each command to execute."""</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">executor</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> :param executor: the executor instance to implement.</span>
+<span class="sd"> :type executor: LocalExecutor</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">executor</span> <span class="o">=</span> <span class="n">executor</span>
+
+ <span class="k">def</span> <span class="nf">start</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">workers_used</span> <span class="o">=</span> <span class="mi">0</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">workers_active</span> <span class="o">=</span> <span class="mi">0</span>
+
+ <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> :param key: the key to identify the TI</span>
+<span class="sd"> :type key: Tuple(dag_id, task_id, execution_date)</span>
+<span class="sd"> :param command: the command to execute</span>
+<span class="sd"> :type command: string</span>
+<span class="sd"> """</span>
+ <span class="n">local_worker</span> <span class="o">=</span> <span class="n">LocalWorker</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">result_queue</span><span class="p">)</span>
+ <span class="n">local_worker</span><span class="o">.</span><span class="n">key</span> <span class="o">=</span> <span class="n">key</span>
+ <span class="n">local_worker</span><span class="o">.</span><span class="n">command</span> <span class="o">=</span> <span class="n">command</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">workers_used</span> <span class="o">+=</span> <span class="mi">1</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">workers_active</span> <span class="o">+=</span> <span class="mi">1</span>
+ <span class="n">local_worker</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
+
+ <span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">empty</span><span class="p">():</span>
+ <span class="n">results</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">change_state</span><span class="p">(</span><span class="o">*</span><span class="n">results</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">workers_active</span> <span class="o">-=</span> <span class="mi">1</span>
+
+ <span class="k">def</span> <span class="nf">end</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">while</span> <span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">workers_active</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">sync</span><span class="p">()</span>
+ <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mf">0.5</span><span class="p">)</span>
+
+ <span class="k">class</span> <span class="nc">_LimitedParallelism</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+ <span class="sd">"""Implements LocalExecutor with limited parallelism using a task queue to</span>
+<span class="sd"> coordinate work distribution."""</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">executor</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">executor</span> <span class="o">=</span> <span class="n">executor</span>
+
+ <span class="k">def</span> <span class="nf">start</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">queue</span> <span class="o">=</span> <span class="n">multiprocessing</span><span class="o">.</span><span class="n">JoinableQueue</span><span class="p">()</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">workers</span> <span class="o">=</span> <span class="p">[</span>
+ <span class="n">QueuedLocalWorker</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">queue</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">result_queue</span><span class="p">)</span>
+ <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">parallelism</span><span class="p">)</span>
+ <span class="p">]</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">workers_used</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">workers</span><span class="p">)</span>
+
+ <span class="k">for</span> <span class="n">w</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">workers</span><span class="p">:</span>
+ <span class="n">w</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
+
+ <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> :param key: the key to identify the TI</span>
+<span class="sd"> :type key: Tuple(dag_id, task_id, execution_date)</span>
+<span class="sd"> :param command: the command to execute</span>
+<span class="sd"> :type command: string</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">))</span>
+
+ <span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">empty</span><span class="p">():</span>
+ <span class="n">results</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">change_state</span><span class="p">(</span><span class="o">*</span><span class="n">results</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">end</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="c1"># Sending poison pill to all worker</span>
+ <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">workers</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">))</span>
+
+ <span class="c1"># Wait for commands to finish</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="o">.</span><span class="n">sync</span><span class="p">()</span>
+
<span class="k">def</span> <span class="nf">start</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">queue</span> <span class="o">=</span> <span class="n">multiprocessing</span><span class="o">.</span><span class="n">JoinableQueue</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="n">multiprocessing</span><span class="o">.</span><span class="n">Queue</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">workers</span> <span class="o">=</span> <span class="p">[</span>
- <span class="n">LocalWorker</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="p">)</span>
- <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">parallelism</span><span class="p">)</span>
- <span class="p">]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">queue</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">workers</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">workers_used</span> <span class="o">=</span> <span class="mi">0</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">workers_active</span> <span class="o">=</span> <span class="mi">0</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">impl</span> <span class="o">=</span> <span class="p">(</span><span class="n">LocalExecutor</span><span class="o">.</span><span class="n">_UnlimitedParallelism</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">parallelism</span> <span class="o">==</span> <span class="mi">0</span>
+ <span class="k">else</span> <span class="n">LocalExecutor</span><span class="o">.</span><span class="n">_LimitedParallelism</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span>
- <span class="k">for</span> <span class="n">w</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">workers</span><span class="p">:</span>
- <span class="n">w</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">impl</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
- <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">))</span>
+ <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">executor_config</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">impl</span><span class="o">.</span><span class="n">execute_async</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="o">=</span><span class="n">command</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">empty</span><span class="p">():</span>
- <span class="n">results</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">change_state</span><span class="p">(</span><span class="o">*</span><span class="n">results</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">impl</span><span class="o">.</span><span class="n">sync</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">end</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="c1"># Sending poison pill to all worker</span>
- <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">workers</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">))</span>
-
- <span class="c1"># Wait for commands to finish</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">sync</span><span class="p">()</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="n">impl</span><span class="o">.</span><span class="n">end</span><span class="p">()</span></div>
</pre></div>
</div>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/executors/sequential_executor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/executors/sequential_executor.html b/_modules/airflow/executors/sequential_executor.html
index 2c7dbac..41373f5 100644
--- a/_modules/airflow/executors/sequential_executor.html
+++ b/_modules/airflow/executors/sequential_executor.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,22 @@
<h1>Source code for airflow.executors.sequential_executor</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
<span class="kn">import</span> <span class="nn">subprocess</span>
@@ -188,7 +195,7 @@
<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span>
-<div class="viewcode-block" id="SequentialExecutor"><a class="viewcode-back" href="../../../code.html#airflow.executors.SequentialExecutor">[docs]</a><span class="k">class</span> <span class="nc">SequentialExecutor</span><span class="p">(</span><span class="n">BaseExecutor</span><span class="p">):</span>
+<div class="viewcode-block" id="SequentialExecutor"><a class="viewcode-back" href="../../../code.html#airflow.executors.sequential_executor.SequentialExecutor">[docs]</a><span class="k">class</span> <span class="nc">SequentialExecutor</span><span class="p">(</span><span class="n">BaseExecutor</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> This executor will only run one task instance at a time, can be used</span>
<span class="sd"> for debugging. It is also the only executor that can be used with sqlite</span>
@@ -201,7 +208,7 @@
<span class="nb">super</span><span class="p">(</span><span class="n">SequentialExecutor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">commands_to_run</span> <span class="o">=</span> <span class="p">[]</span>
- <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">executor_config</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">commands_to_run</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,))</span>
<span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
@@ -209,7 +216,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Executing command: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">command</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
- <span class="n">subprocess</span><span class="o">.</span><span class="n">check_call</span><span class="p">(</span><span class="n">command</span><span class="p">,</span> <span class="n">shell</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="n">subprocess</span><span class="o">.</span><span class="n">check_call</span><span class="p">(</span><span class="n">command</span><span class="p">,</span> <span class="n">shell</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">close_fds</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">change_state</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">)</span>
<span class="k">except</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">CalledProcessError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">change_state</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">FAILED</span><span class="p">)</span>
[02/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/sensors/web_hdfs_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/sensors/web_hdfs_sensor.html b/_modules/airflow/sensors/web_hdfs_sensor.html
new file mode 100644
index 0000000..32711ea
--- /dev/null
+++ b/_modules/airflow/sensors/web_hdfs_sensor.html
@@ -0,0 +1,279 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.sensors.web_hdfs_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.sensors.web_hdfs_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.sensors.web_hdfs_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.sensors.base_sensor_operator</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="WebHdfsSensor"><a class="viewcode-back" href="../../../code.html#airflow.sensors.web_hdfs_sensor.WebHdfsSensor">[docs]</a><span class="k">class</span> <span class="nc">WebHdfsSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Waits for a file or folder to land in HDFS</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'filepath'</span><span class="p">,)</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">filepath</span><span class="p">,</span>
+ <span class="n">webhdfs_conn_id</span><span class="o">=</span><span class="s1">'webhdfs_default'</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">WebHdfsSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">filepath</span> <span class="o">=</span> <span class="n">filepath</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">webhdfs_conn_id</span> <span class="o">=</span> <span class="n">webhdfs_conn_id</span>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.webhdfs_hook</span> <span class="k">import</span> <span class="n">WebHDFSHook</span>
+ <span class="n">c</span> <span class="o">=</span> <span class="n">WebHDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">webhdfs_conn_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Poking for file </span><span class="si">{self.filepath}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
+ <span class="k">return</span> <span class="n">c</span><span class="o">.</span><span class="n">check_for_path</span><span class="p">(</span><span class="n">hdfs_path</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/bash_operator.html
----------------------------------------------------------------------
diff --git a/_modules/bash_operator.html b/_modules/bash_operator.html
deleted file mode 100644
index 3b6f9ef..0000000
--- a/_modules/bash_operator.html
+++ /dev/null
@@ -1,350 +0,0 @@
-
-
-<!DOCTYPE html>
-<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
-<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
-<head>
- <meta charset="utf-8">
-
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
-
- <title>bash_operator — Airflow Documentation</title>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../genindex.html"/>
- <link rel="search" title="Search" href="../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../index.html"/>
- <link rel="up" title="Module code" href="index.html"/>
-
-
- <script src="../_static/js/modernizr.min.js"></script>
-
-</head>
-
-<body class="wy-body-for-nav" role="document">
-
-
- <div class="wy-grid-for-nav">
-
-
- <nav data-toggle="wy-nav-shift" class="wy-nav-side">
- <div class="wy-side-scroll">
- <div class="wy-side-nav-search">
-
-
-
- <a href="../index.html" class="icon icon-home"> Airflow
-
-
-
- </a>
-
-
-
-
-
-
-
-<div role="search">
- <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
- <input type="text" name="q" placeholder="Search docs" />
- <input type="hidden" name="check_keywords" value="yes" />
- <input type="hidden" name="area" value="default" />
- </form>
-</div>
-
-
- </div>
-
- <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
-
-
-
-
-
-
- <ul>
-<li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../configuration.html">Configuration</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../ui.html">UI / Screenshots</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../concepts.html">Concepts</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../profiling.html">Data Profiling</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../cli.html">Command Line Interface</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../scheduler.html">Scheduling & Triggers</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../plugins.html">Plugins</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../api.html">Experimental Rest API</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../integration.html">Integration</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../code.html">API Reference</a></li>
-</ul>
-
-
-
- </div>
- </div>
- </nav>
-
- <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
-
-
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
-
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="../index.html">Airflow</a>
-
- </nav>
-
-
-
- <div class="wy-nav-content">
- <div class="rst-content">
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-<div role="navigation" aria-label="breadcrumbs navigation">
-
- <ul class="wy-breadcrumbs">
-
- <li><a href="../index.html">Docs</a> »</li>
-
- <li><a href="index.html">Module code</a> »</li>
-
- <li>bash_operator</li>
-
-
- <li class="wy-breadcrumbs-aside">
-
-
-
- </li>
-
- </ul>
-
-
- <hr/>
-</div>
- <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
- <div itemprop="articleBody">
-
- <h1>Source code for bash_operator</h1><div class="highlight"><pre>
-<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
-<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
-
-
-<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">bytes</span>
-<span class="kn">import</span> <span class="nn">os</span>
-<span class="kn">import</span> <span class="nn">signal</span>
-<span class="kn">from</span> <span class="nn">subprocess</span> <span class="k">import</span> <span class="n">Popen</span><span class="p">,</span> <span class="n">STDOUT</span><span class="p">,</span> <span class="n">PIPE</span>
-<span class="kn">from</span> <span class="nn">tempfile</span> <span class="k">import</span> <span class="n">gettempdir</span><span class="p">,</span> <span class="n">NamedTemporaryFile</span>
-
-<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
-<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.file</span> <span class="k">import</span> <span class="n">TemporaryDirectory</span>
-
-
-<div class="viewcode-block" id="BashOperator"><a class="viewcode-back" href="../code.html#airflow.operators.BashOperator">[docs]</a><span class="k">class</span> <span class="nc">BashOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Execute a Bash script, command or set of commands.</span>
-
-<span class="sd"> :param bash_command: The command, set of commands or reference to a</span>
-<span class="sd"> bash script (must be '.sh') to be executed.</span>
-<span class="sd"> :type bash_command: string</span>
-<span class="sd"> :param xcom_push: If xcom_push is True, the last line written to stdout</span>
-<span class="sd"> will also be pushed to an XCom when the bash command completes.</span>
-<span class="sd"> :type xcom_push: bool</span>
-<span class="sd"> :param env: If env is not None, it must be a mapping that defines the</span>
-<span class="sd"> environment variables for the new process; these are used instead</span>
-<span class="sd"> of inheriting the current process environment, which is the default</span>
-<span class="sd"> behavior. (templated)</span>
-<span class="sd"> :type env: dict</span>
-<span class="sd"> :type output_encoding: output encoding of bash command</span>
-<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bash_command'</span><span class="p">,</span> <span class="s1">'env'</span><span class="p">)</span>
- <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sh'</span><span class="p">,</span> <span class="s1">'.bash'</span><span class="p">,)</span>
- <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f0ede4'</span>
-
- <span class="nd">@apply_defaults</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
- <span class="bp">self</span><span class="p">,</span>
- <span class="n">bash_command</span><span class="p">,</span>
- <span class="n">xcom_push</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">env</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">output_encoding</span><span class="o">=</span><span class="s1">'utf-8'</span><span class="p">,</span>
- <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
-
- <span class="nb">super</span><span class="p">(</span><span class="n">BashOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">bash_command</span> <span class="o">=</span> <span class="n">bash_command</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">env</span> <span class="o">=</span> <span class="n">env</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">xcom_push_flag</span> <span class="o">=</span> <span class="n">xcom_push</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">output_encoding</span> <span class="o">=</span> <span class="n">output_encoding</span>
-
-<div class="viewcode-block" id="BashOperator.execute"><a class="viewcode-back" href="../code.html#airflow.operators.BashOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Execute the bash command in a temporary directory</span>
-<span class="sd"> which will be cleaned afterwards</span>
-<span class="sd"> """</span>
- <span class="n">bash_command</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">bash_command</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Tmp dir root location: </span><span class="se">\n</span><span class="s2"> </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">gettempdir</span><span class="p">())</span>
- <span class="k">with</span> <span class="n">TemporaryDirectory</span><span class="p">(</span><span class="n">prefix</span><span class="o">=</span><span class="s1">'airflowtmp'</span><span class="p">)</span> <span class="k">as</span> <span class="n">tmp_dir</span><span class="p">:</span>
- <span class="k">with</span> <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="nb">dir</span><span class="o">=</span><span class="n">tmp_dir</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
-
- <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="nb">bytes</span><span class="p">(</span><span class="n">bash_command</span><span class="p">,</span> <span class="s1">'utf_8'</span><span class="p">))</span>
- <span class="n">f</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
- <span class="n">fname</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">name</span>
- <span class="n">script_location</span> <span class="o">=</span> <span class="n">tmp_dir</span> <span class="o">+</span> <span class="s2">"/"</span> <span class="o">+</span> <span class="n">fname</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
- <span class="s2">"Temporary script location: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
- <span class="n">script_location</span>
- <span class="p">)</span>
- <span class="k">def</span> <span class="nf">pre_exec</span><span class="p">():</span>
- <span class="c1"># Restore default signal disposition and invoke setsid</span>
- <span class="k">for</span> <span class="n">sig</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'SIGPIPE'</span><span class="p">,</span> <span class="s1">'SIGXFZ'</span><span class="p">,</span> <span class="s1">'SIGXFSZ'</span><span class="p">):</span>
- <span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">signal</span><span class="p">,</span> <span class="n">sig</span><span class="p">):</span>
- <span class="n">signal</span><span class="o">.</span><span class="n">signal</span><span class="p">(</span><span class="nb">getattr</span><span class="p">(</span><span class="n">signal</span><span class="p">,</span> <span class="n">sig</span><span class="p">),</span> <span class="n">signal</span><span class="o">.</span><span class="n">SIG_DFL</span><span class="p">)</span>
- <span class="n">os</span><span class="o">.</span><span class="n">setsid</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Running command: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">bash_command</span><span class="p">)</span>
- <span class="n">sp</span> <span class="o">=</span> <span class="n">Popen</span><span class="p">(</span>
- <span class="p">[</span><span class="s1">'bash'</span><span class="p">,</span> <span class="n">fname</span><span class="p">],</span>
- <span class="n">stdout</span><span class="o">=</span><span class="n">PIPE</span><span class="p">,</span> <span class="n">stderr</span><span class="o">=</span><span class="n">STDOUT</span><span class="p">,</span>
- <span class="n">cwd</span><span class="o">=</span><span class="n">tmp_dir</span><span class="p">,</span> <span class="n">env</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">env</span><span class="p">,</span>
- <span class="n">preexec_fn</span><span class="o">=</span><span class="n">pre_exec</span><span class="p">)</span>
-
- <span class="bp">self</span><span class="o">.</span><span class="n">sp</span> <span class="o">=</span> <span class="n">sp</span>
-
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Output:"</span><span class="p">)</span>
- <span class="n">line</span> <span class="o">=</span> <span class="s1">''</span>
- <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="nb">iter</span><span class="p">(</span><span class="n">sp</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">readline</span><span class="p">,</span> <span class="sa">b</span><span class="s1">''</span><span class="p">):</span>
- <span class="n">line</span> <span class="o">=</span> <span class="n">line</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">output_encoding</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">line</span><span class="p">)</span>
- <span class="n">sp</span><span class="o">.</span><span class="n">wait</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
- <span class="s2">"Command exited with return code </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
- <span class="n">sp</span><span class="o">.</span><span class="n">returncode</span>
- <span class="p">)</span>
-
- <span class="k">if</span> <span class="n">sp</span><span class="o">.</span><span class="n">returncode</span><span class="p">:</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Bash command failed"</span><span class="p">)</span>
-
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">xcom_push_flag</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">line</span></div>
-
- <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Sending SIGTERM signal to bash process group'</span><span class="p">)</span>
- <span class="n">os</span><span class="o">.</span><span class="n">killpg</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">getpgid</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sp</span><span class="o">.</span><span class="n">pid</span><span class="p">),</span> <span class="n">signal</span><span class="o">.</span><span class="n">SIGTERM</span><span class="p">)</span></div>
-
-</pre></div>
-
- </div>
- <div class="articleComments">
-
- </div>
- </div>
- <footer>
-
-
- <hr/>
-
- <div role="contentinfo">
- <p>
-
- </p>
- </div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
-
-</footer>
-
- </div>
- </div>
-
- </section>
-
- </div>
-
-
-
-
-
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../',
- VERSION:'',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../_static/jquery.js"></script>
- <script type="text/javascript" src="../_static/underscore.js"></script>
- <script type="text/javascript" src="../_static/doctools.js"></script>
-
-
-
-
-
- <script type="text/javascript" src="../_static/js/theme.js"></script>
-
-
-
-
- <script type="text/javascript">
- jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
- });
- </script>
-
-
-</body>
-</html>
\ No newline at end of file
[49/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/bigquery_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/bigquery_hook.html b/_modules/airflow/contrib/hooks/bigquery_hook.html
index 1926c79..aff2ebd 100644
--- a/_modules/airflow/contrib/hooks/bigquery_hook.html
+++ b/_modules/airflow/contrib/hooks/bigquery_hook.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -169,39 +171,45 @@
<h1>Source code for airflow.contrib.hooks.bigquery_hook</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
-<span class="c1">#</span>
-
<span class="sd">"""</span>
<span class="sd">This module contains a BigQuery Hook, as well as a very basic PEP 249</span>
<span class="sd">implementation for BigQuery.</span>
<span class="sd">"""</span>
<span class="kn">import</span> <span class="nn">time</span>
-
-<span class="kn">from</span> <span class="nn">apiclient.discovery</span> <span class="k">import</span> <span class="n">build</span><span class="p">,</span> <span class="n">HttpError</span>
-<span class="kn">from</span> <span class="nn">googleapiclient</span> <span class="k">import</span> <span class="n">errors</span>
<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">range</span>
-<span class="kn">from</span> <span class="nn">pandas_gbq.gbq</span> <span class="k">import</span> <span class="n">GbqConnector</span><span class="p">,</span> \
- <span class="n">_parse_data</span> <span class="k">as</span> <span class="n">gbq_parse_data</span><span class="p">,</span> \
- <span class="n">_check_google_client_version</span> <span class="k">as</span> <span class="n">gbq_check_google_client_version</span><span class="p">,</span> \
- <span class="n">_test_google_api_imports</span> <span class="k">as</span> <span class="n">gbq_test_google_api_imports</span>
-<span class="kn">from</span> <span class="nn">pandas.tools.merge</span> <span class="k">import</span> <span class="n">concat</span>
+
<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="k">import</span> <span class="n">basestring</span>
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">AirflowException</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="k">import</span> <span class="n">GoogleCloudBaseHook</span>
<span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="k">import</span> <span class="n">DbApiHook</span>
<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+<span class="kn">from</span> <span class="nn">apiclient.discovery</span> <span class="k">import</span> <span class="n">HttpError</span><span class="p">,</span> <span class="n">build</span>
+<span class="kn">from</span> <span class="nn">googleapiclient</span> <span class="k">import</span> <span class="n">errors</span>
+<span class="kn">from</span> <span class="nn">pandas_gbq.gbq</span> <span class="k">import</span> \
+ <span class="n">_check_google_client_version</span> <span class="k">as</span> <span class="n">gbq_check_google_client_version</span>
+<span class="kn">from</span> <span class="nn">pandas_gbq</span> <span class="k">import</span> <span class="n">read_gbq</span>
+<span class="kn">from</span> <span class="nn">pandas_gbq.gbq</span> <span class="k">import</span> \
+ <span class="n">_test_google_api_imports</span> <span class="k">as</span> <span class="n">gbq_test_google_api_imports</span>
+<span class="kn">from</span> <span class="nn">pandas_gbq.gbq</span> <span class="k">import</span> <span class="n">GbqConnector</span>
<div class="viewcode-block" id="BigQueryHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook">[docs]</a><span class="k">class</span> <span class="nc">BigQueryHook</span><span class="p">(</span><span class="n">GoogleCloudBaseHook</span><span class="p">,</span> <span class="n">DbApiHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
@@ -213,10 +221,11 @@
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
<span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">'bigquery_default'</span><span class="p">,</span>
- <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">use_legacy_sql</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">BigQueryHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
- <span class="n">conn_id</span><span class="o">=</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
- <span class="n">delegate_to</span><span class="o">=</span><span class="n">delegate_to</span><span class="p">)</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="n">bigquery_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="o">=</span><span class="n">delegate_to</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span> <span class="o">=</span> <span class="n">use_legacy_sql</span>
<div class="viewcode-block" id="BigQueryHook.get_conn"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -224,7 +233,10 @@
<span class="sd"> """</span>
<span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_service</span><span class="p">()</span>
<span class="n">project</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_field</span><span class="p">(</span><span class="s1">'project'</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">BigQueryConnection</span><span class="p">(</span><span class="n">service</span><span class="o">=</span><span class="n">service</span><span class="p">,</span> <span class="n">project_id</span><span class="o">=</span><span class="n">project</span><span class="p">)</span></div>
+ <span class="k">return</span> <span class="n">BigQueryConnection</span><span class="p">(</span>
+ <span class="n">service</span><span class="o">=</span><span class="n">service</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="o">=</span><span class="n">project</span><span class="p">,</span>
+ <span class="n">use_legacy_sql</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span><span class="p">)</span></div>
<div class="viewcode-block" id="BigQueryHook.get_service"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook.get_service">[docs]</a> <span class="k">def</span> <span class="nf">get_service</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -241,7 +253,7 @@
<span class="sd"> """</span>
<span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">()</span></div>
-<div class="viewcode-block" id="BigQueryHook.get_pandas_df"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook.get_pandas_df">[docs]</a> <span class="k">def</span> <span class="nf">get_pandas_df</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">dialect</span><span class="o">=</span><span class="s1">'legacy'</span><span class="p">):</span>
+<div class="viewcode-block" id="BigQueryHook.get_pandas_df"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook.get_pandas_df">[docs]</a> <span class="k">def</span> <span class="nf">get_pandas_df</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">dialect</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Returns a Pandas DataFrame for the results produced by a BigQuery</span>
<span class="sd"> query. The DbApiHook method must be overridden because Pandas</span>
@@ -252,35 +264,31 @@
<span class="sd"> :param bql: The BigQuery SQL to execute.</span>
<span class="sd"> :type bql: string</span>
-<span class="sd"> :param parameters: The parameters to render the SQL query with (not used, leave to override superclass method)</span>
+<span class="sd"> :param parameters: The parameters to render the SQL query with (not</span>
+<span class="sd"> used, leave to override superclass method)</span>
<span class="sd"> :type parameters: mapping or iterable</span>
<span class="sd"> :param dialect: Dialect of BigQuery SQL – legacy SQL or standard SQL</span>
-<span class="sd"> :type dialect: string in {'legacy', 'standard'}, default 'legacy'</span>
+<span class="sd"> defaults to use `self.use_legacy_sql` if not specified</span>
+<span class="sd"> :type dialect: string in {'legacy', 'standard'}</span>
<span class="sd"> """</span>
- <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_service</span><span class="p">()</span>
- <span class="n">project</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_field</span><span class="p">(</span><span class="s1">'project'</span><span class="p">)</span>
- <span class="n">connector</span> <span class="o">=</span> <span class="n">BigQueryPandasConnector</span><span class="p">(</span><span class="n">project</span><span class="p">,</span> <span class="n">service</span><span class="p">,</span> <span class="n">dialect</span><span class="o">=</span><span class="n">dialect</span><span class="p">)</span>
- <span class="n">schema</span><span class="p">,</span> <span class="n">pages</span> <span class="o">=</span> <span class="n">connector</span><span class="o">.</span><span class="n">run_query</span><span class="p">(</span><span class="n">bql</span><span class="p">)</span>
- <span class="n">dataframe_list</span> <span class="o">=</span> <span class="p">[]</span>
-
- <span class="k">while</span> <span class="nb">len</span><span class="p">(</span><span class="n">pages</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
- <span class="n">page</span> <span class="o">=</span> <span class="n">pages</span><span class="o">.</span><span class="n">pop</span><span class="p">()</span>
- <span class="n">dataframe_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">gbq_parse_data</span><span class="p">(</span><span class="n">schema</span><span class="p">,</span> <span class="n">page</span><span class="p">))</span>
+ <span class="k">if</span> <span class="n">dialect</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">dialect</span> <span class="o">=</span> <span class="s1">'legacy'</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span> <span class="k">else</span> <span class="s1">'standard'</span>
- <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">dataframe_list</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">concat</span><span class="p">(</span><span class="n">dataframe_list</span><span class="p">,</span> <span class="n">ignore_index</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">gbq_parse_data</span><span class="p">(</span><span class="n">schema</span><span class="p">,</span> <span class="p">[])</span></div>
+ <span class="k">return</span> <span class="n">read_gbq</span><span class="p">(</span><span class="n">bql</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_get_field</span><span class="p">(</span><span class="s1">'project'</span><span class="p">),</span>
+ <span class="n">dialect</span><span class="o">=</span><span class="n">dialect</span><span class="p">,</span>
+ <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
<div class="viewcode-block" id="BigQueryHook.table_exists"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook.table_exists">[docs]</a> <span class="k">def</span> <span class="nf">table_exists</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Checks for the existence of a table in Google BigQuery.</span>
-<span class="sd"> :param project_id: The Google cloud project in which to look for the table. The connection supplied to the hook</span>
-<span class="sd"> must provide access to the specified project.</span>
+<span class="sd"> :param project_id: The Google cloud project in which to look for the</span>
+<span class="sd"> table. The connection supplied to the hook must provide access to</span>
+<span class="sd"> the specified project.</span>
<span class="sd"> :type project_id: string</span>
-<span class="sd"> :param dataset_id: The name of the dataset in which to look for the table.</span>
-<span class="sd"> storage bucket.</span>
+<span class="sd"> :param dataset_id: The name of the dataset in which to look for the</span>
+<span class="sd"> table.</span>
<span class="sd"> :type dataset_id: string</span>
<span class="sd"> :param table_id: The name of the table to check the existence of.</span>
<span class="sd"> :type table_id: string</span>
@@ -288,10 +296,8 @@
<span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_service</span><span class="p">()</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
- <span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
- <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
- <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span>
- <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span> <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+ <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
<span class="k">return</span> <span class="kc">True</span>
<span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="k">if</span> <span class="n">e</span><span class="o">.</span><span class="n">resp</span><span class="p">[</span><span class="s1">'status'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'404'</span><span class="p">:</span>
@@ -307,7 +313,14 @@
<span class="sd"> without forcing a three legged OAuth connection. Instead, we can inject</span>
<span class="sd"> service account credentials into the binding.</span>
<span class="sd"> """</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">service</span><span class="p">,</span> <span class="n">reauth</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">dialect</span><span class="o">=</span><span class="s1">'legacy'</span><span class="p">):</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span>
+ <span class="n">service</span><span class="p">,</span>
+ <span class="n">reauth</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">dialect</span><span class="o">=</span><span class="s1">'legacy'</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryPandasConnector</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">project_id</span><span class="p">)</span>
<span class="n">gbq_check_google_client_version</span><span class="p">()</span>
<span class="n">gbq_test_google_api_imports</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
@@ -351,19 +364,280 @@
<span class="sd"> BigQuery. The methods can be used directly by operators, in cases where a</span>
<span class="sd"> PEP 249 cursor isn't needed.</span>
<span class="sd"> """</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">service</span><span class="p">,</span> <span class="n">project_id</span><span class="p">):</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">service</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">use_legacy_sql</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">service</span> <span class="o">=</span> <span class="n">service</span>
<span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span> <span class="o">=</span> <span class="n">use_legacy_sql</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span> <span class="o">=</span> <span class="kc">None</span>
+
+ <span class="k">def</span> <span class="nf">create_empty_table</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span>
+ <span class="n">dataset_id</span><span class="p">,</span>
+ <span class="n">table_id</span><span class="p">,</span>
+ <span class="n">schema_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">time_partitioning</span><span class="o">=</span><span class="p">{}</span>
+ <span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Creates a new, empty table in the dataset.</span>
+
+<span class="sd"> :param project_id: The project to create the table into.</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :param dataset_id: The dataset to create the table into.</span>
+<span class="sd"> :type dataset_id: str</span>
+<span class="sd"> :param table_id: The Name of the table to be created.</span>
+<span class="sd"> :type table_id: str</span>
+<span class="sd"> :param schema_fields: If set, the schema field list as defined here:</span>
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load.schema</span>
+
+<span class="sd"> **Example**: ::</span>
+
+<span class="sd"> schema_fields=[{"name": "emp_name", "type": "STRING", "mode": "REQUIRED"},</span>
+<span class="sd"> {"name": "salary", "type": "INTEGER", "mode": "NULLABLE"}]</span>
+
+<span class="sd"> :type schema_fields: list</span>
+<span class="sd"> :param time_partitioning: configure optional time partitioning fields i.e.</span>
+<span class="sd"> partition by field, type and expiration as per API specifications.</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#timePartitioning</span>
+<span class="sd"> :type time_partitioning: dict</span>
+
+<span class="sd"> :return:</span>
+<span class="sd"> """</span>
+ <span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span> <span class="k">if</span> <span class="n">project_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
+
+ <span class="n">table_resource</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'tableReference'</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s1">'tableId'</span><span class="p">:</span> <span class="n">table_id</span>
+ <span class="p">}</span>
+ <span class="p">}</span>
+
+ <span class="k">if</span> <span class="n">schema_fields</span><span class="p">:</span>
+ <span class="n">table_resource</span><span class="p">[</span><span class="s1">'schema'</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'fields'</span><span class="p">:</span> <span class="n">schema_fields</span><span class="p">}</span>
+
+ <span class="k">if</span> <span class="n">time_partitioning</span><span class="p">:</span>
+ <span class="n">table_resource</span><span class="p">[</span><span class="s1">'timePartitioning'</span><span class="p">]</span> <span class="o">=</span> <span class="n">time_partitioning</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Creating Table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+ <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Table created successfully: </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
+
+ <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'BigQuery job failed. Error was: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">err</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
+ <span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">create_external_table</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">external_project_dataset_table</span><span class="p">,</span>
+ <span class="n">schema_fields</span><span class="p">,</span>
+ <span class="n">source_uris</span><span class="p">,</span>
+ <span class="n">source_format</span><span class="o">=</span><span class="s1">'CSV'</span><span class="p">,</span>
+ <span class="n">autodetect</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">compression</span><span class="o">=</span><span class="s1">'NONE'</span><span class="p">,</span>
+ <span class="n">ignore_unknown_values</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">max_bad_records</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
+ <span class="n">skip_leading_rows</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
+ <span class="n">field_delimiter</span><span class="o">=</span><span class="s1">','</span><span class="p">,</span>
+ <span class="n">quote_character</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">allow_quoted_newlines</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">allow_jagged_rows</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">src_fmt_configs</span><span class="o">=</span><span class="p">{}</span>
+ <span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Creates a new external table in the dataset with the data in Google</span>
+<span class="sd"> Cloud Storage. See here:</span>
+
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#resource</span>
+
+<span class="sd"> for more details about these parameters.</span>
+
+<span class="sd"> :param external_project_dataset_table:</span>
+<span class="sd"> The dotted (<project>.|<project>:)<dataset>.<table>($<partition>) BigQuery</span>
+<span class="sd"> table name to create external table.</span>
+<span class="sd"> If <project> is not included, project will be the</span>
+<span class="sd"> project defined in the connection json.</span>
+<span class="sd"> :type external_project_dataset_table: string</span>
+<span class="sd"> :param schema_fields: The schema field list as defined here:</span>
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#resource</span>
+<span class="sd"> :type schema_fields: list</span>
+<span class="sd"> :param source_uris: The source Google Cloud</span>
+<span class="sd"> Storage URI (e.g. gs://some-bucket/some-file.txt). A single wild</span>
+<span class="sd"> per-object name can be used.</span>
+<span class="sd"> :type source_uris: list</span>
+<span class="sd"> :param source_format: File format to export.</span>
+<span class="sd"> :type source_format: string</span>
+<span class="sd"> :param autodetect: Try to detect schema and format options automatically.</span>
+<span class="sd"> Any option specified explicitly will be honored.</span>
+<span class="sd"> :type autodetect: bool</span>
+<span class="sd"> :param compression: [Optional] The compression type of the data source.</span>
+<span class="sd"> Possible values include GZIP and NONE.</span>
+<span class="sd"> The default value is NONE.</span>
+<span class="sd"> This setting is ignored for Google Cloud Bigtable,</span>
+<span class="sd"> Google Cloud Datastore backups and Avro formats.</span>
+<span class="sd"> :type compression: string</span>
+<span class="sd"> :param ignore_unknown_values: [Optional] Indicates if BigQuery should allow</span>
+<span class="sd"> extra values that are not represented in the table schema.</span>
+<span class="sd"> If true, the extra values are ignored. If false, records with extra columns</span>
+<span class="sd"> are treated as bad records, and if there are too many bad records, an</span>
+<span class="sd"> invalid error is returned in the job result.</span>
+<span class="sd"> :type ignore_unknown_values: bool</span>
+<span class="sd"> :param max_bad_records: The maximum number of bad records that BigQuery can</span>
+<span class="sd"> ignore when running the job.</span>
+<span class="sd"> :type max_bad_records: int</span>
+<span class="sd"> :param skip_leading_rows: Number of rows to skip when loading from a CSV.</span>
+<span class="sd"> :type skip_leading_rows: int</span>
+<span class="sd"> :param field_delimiter: The delimiter to use when loading from a CSV.</span>
+<span class="sd"> :type field_delimiter: string</span>
+<span class="sd"> :param quote_character: The value that is used to quote data sections in a CSV</span>
+<span class="sd"> file.</span>
+<span class="sd"> :type quote_character: string</span>
+<span class="sd"> :param allow_quoted_newlines: Whether to allow quoted newlines (true) or not</span>
+<span class="sd"> (false).</span>
+<span class="sd"> :type allow_quoted_newlines: boolean</span>
+<span class="sd"> :param allow_jagged_rows: Accept rows that are missing trailing optional columns.</span>
+<span class="sd"> The missing values are treated as nulls. If false, records with missing</span>
+<span class="sd"> trailing columns are treated as bad records, and if there are too many bad</span>
+<span class="sd"> records, an invalid error is returned in the job result. Only applicable when</span>
+<span class="sd"> soure_format is CSV.</span>
+<span class="sd"> :type allow_jagged_rows: bool</span>
+<span class="sd"> :param src_fmt_configs: configure optional fields specific to the source format</span>
+<span class="sd"> :type src_fmt_configs: dict</span>
+<span class="sd"> """</span>
+
+ <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">external_table_id</span> <span class="o">=</span> \
+ <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">external_project_dataset_table</span><span class="p">,</span>
+ <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">var_name</span><span class="o">=</span><span class="s1">'external_project_dataset_table'</span><span class="p">)</span>
+
+ <span class="c1"># bigquery only allows certain source formats</span>
+ <span class="c1"># we check to make sure the passed source format is valid</span>
+ <span class="c1"># if it's not, we raise a ValueError</span>
+ <span class="c1"># Refer to this link for more details:</span>
+ <span class="c1"># https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#externalDataConfiguration.sourceFormat</span>
+
+ <span class="n">source_format</span> <span class="o">=</span> <span class="n">source_format</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
+ <span class="n">allowed_formats</span> <span class="o">=</span> <span class="p">[</span>
+ <span class="s2">"CSV"</span><span class="p">,</span> <span class="s2">"NEWLINE_DELIMITED_JSON"</span><span class="p">,</span> <span class="s2">"AVRO"</span><span class="p">,</span> <span class="s2">"GOOGLE_SHEETS"</span><span class="p">,</span>
+ <span class="s2">"DATASTORE_BACKUP"</span><span class="p">,</span> <span class="s2">"PARQUET"</span>
+ <span class="p">]</span>
+ <span class="k">if</span> <span class="n">source_format</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">allowed_formats</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"</span><span class="si">{0}</span><span class="s2"> is not a valid source format. "</span>
+ <span class="s2">"Please use one of the following types: </span><span class="si">{1}</span><span class="s2">"</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">source_format</span><span class="p">,</span> <span class="n">allowed_formats</span><span class="p">))</span>
+
+ <span class="n">compression</span> <span class="o">=</span> <span class="n">compression</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
+ <span class="n">allowed_compressions</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'NONE'</span><span class="p">,</span> <span class="s1">'GZIP'</span><span class="p">]</span>
+ <span class="k">if</span> <span class="n">compression</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">allowed_compressions</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"</span><span class="si">{0}</span><span class="s2"> is not a valid compression format. "</span>
+ <span class="s2">"Please use one of the following types: </span><span class="si">{1}</span><span class="s2">"</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">compression</span><span class="p">,</span> <span class="n">allowed_compressions</span><span class="p">))</span>
+
+ <span class="n">table_resource</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'externalDataConfiguration'</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s1">'autodetect'</span><span class="p">:</span> <span class="n">autodetect</span><span class="p">,</span>
+ <span class="s1">'sourceFormat'</span><span class="p">:</span> <span class="n">source_format</span><span class="p">,</span>
+ <span class="s1">'sourceUris'</span><span class="p">:</span> <span class="n">source_uris</span><span class="p">,</span>
+ <span class="s1">'compression'</span><span class="p">:</span> <span class="n">compression</span><span class="p">,</span>
+ <span class="s1">'ignoreUnknownValues'</span><span class="p">:</span> <span class="n">ignore_unknown_values</span>
+ <span class="p">},</span>
+ <span class="s1">'tableReference'</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s1">'projectId'</span><span class="p">:</span> <span class="n">project_id</span><span class="p">,</span>
+ <span class="s1">'datasetId'</span><span class="p">:</span> <span class="n">dataset_id</span><span class="p">,</span>
+ <span class="s1">'tableId'</span><span class="p">:</span> <span class="n">external_table_id</span><span class="p">,</span>
+ <span class="p">}</span>
+ <span class="p">}</span>
+
+ <span class="k">if</span> <span class="n">schema_fields</span><span class="p">:</span>
+ <span class="n">table_resource</span><span class="p">[</span><span class="s1">'externalDataConfiguration'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">({</span>
+ <span class="s1">'schema'</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s1">'fields'</span><span class="p">:</span> <span class="n">schema_fields</span>
+ <span class="p">}</span>
+ <span class="p">})</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Creating external table: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">external_project_dataset_table</span><span class="p">)</span>
- <span class="k">def</span> <span class="nf">run_query</span><span class="p">(</span>
- <span class="bp">self</span><span class="p">,</span> <span class="n">bql</span><span class="p">,</span> <span class="n">destination_dataset_table</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
- <span class="n">write_disposition</span> <span class="o">=</span> <span class="s1">'WRITE_EMPTY'</span><span class="p">,</span>
- <span class="n">allow_large_results</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">udf_config</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
- <span class="n">use_legacy_sql</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
- <span class="n">maximum_billing_tier</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">create_disposition</span><span class="o">=</span><span class="s1">'CREATE_IF_NEEDED'</span><span class="p">,</span>
- <span class="n">query_params</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">max_bad_records</span><span class="p">:</span>
+ <span class="n">table_resource</span><span class="p">[</span><span class="s1">'externalDataConfiguration'</span><span class="p">][</span><span class="s1">'maxBadRecords'</span><span class="p">]</span> <span class="o">=</span> <span class="n">max_bad_records</span>
+
+ <span class="c1"># if following fields are not specified in src_fmt_configs,</span>
+ <span class="c1"># honor the top-level params for backward-compatibility</span>
+ <span class="k">if</span> <span class="s1">'skipLeadingRows'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
+ <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">'skipLeadingRows'</span><span class="p">]</span> <span class="o">=</span> <span class="n">skip_leading_rows</span>
+ <span class="k">if</span> <span class="s1">'fieldDelimiter'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
+ <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">'fieldDelimiter'</span><span class="p">]</span> <span class="o">=</span> <span class="n">field_delimiter</span>
+ <span class="k">if</span> <span class="s1">'quote_character'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
+ <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">'quote'</span><span class="p">]</span> <span class="o">=</span> <span class="n">quote_character</span>
+ <span class="k">if</span> <span class="s1">'allowQuotedNewlines'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
+ <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">'allowQuotedNewlines'</span><span class="p">]</span> <span class="o">=</span> <span class="n">allow_quoted_newlines</span>
+ <span class="k">if</span> <span class="s1">'allowJaggedRows'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
+ <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">'allowJaggedRows'</span><span class="p">]</span> <span class="o">=</span> <span class="n">allow_jagged_rows</span>
+
+ <span class="n">src_fmt_to_param_mapping</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'CSV'</span><span class="p">:</span> <span class="s1">'csvOptions'</span><span class="p">,</span>
+ <span class="s1">'GOOGLE_SHEETS'</span><span class="p">:</span> <span class="s1">'googleSheetsOptions'</span>
+ <span class="p">}</span>
+
+ <span class="n">src_fmt_to_configs_mapping</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'csvOptions'</span><span class="p">:</span> <span class="p">[</span>
+ <span class="s1">'allowJaggedRows'</span><span class="p">,</span> <span class="s1">'allowQuotedNewlines'</span><span class="p">,</span>
+ <span class="s1">'fieldDelimiter'</span><span class="p">,</span> <span class="s1">'skipLeadingRows'</span><span class="p">,</span>
+ <span class="s1">'quote'</span>
+ <span class="p">],</span>
+ <span class="s1">'googleSheetsOptions'</span><span class="p">:</span> <span class="p">[</span><span class="s1">'skipLeadingRows'</span><span class="p">]</span>
+ <span class="p">}</span>
+
+ <span class="k">if</span> <span class="n">source_format</span> <span class="ow">in</span> <span class="n">src_fmt_to_param_mapping</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
+
+ <span class="n">valid_configs</span> <span class="o">=</span> <span class="n">src_fmt_to_configs_mapping</span><span class="p">[</span>
+ <span class="n">src_fmt_to_param_mapping</span><span class="p">[</span><span class="n">source_format</span><span class="p">]</span>
+ <span class="p">]</span>
+
+ <span class="n">src_fmt_configs</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="n">k</span><span class="p">:</span> <span class="n">v</span>
+ <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="k">if</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">valid_configs</span>
+ <span class="p">}</span>
+
+ <span class="n">table_resource</span><span class="p">[</span><span class="s1">'externalDataConfiguration'</span><span class="p">][</span><span class="n">src_fmt_to_param_mapping</span><span class="p">[</span>
+ <span class="n">source_format</span><span class="p">]]</span> <span class="o">=</span> <span class="n">src_fmt_configs</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+ <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'External table created successfully: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">external_project_dataset_table</span><span class="p">)</span>
+
+ <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
+ <span class="s1">'BigQuery job failed. Error was: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">err</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
+ <span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">run_query</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">bql</span><span class="p">,</span>
+ <span class="n">destination_dataset_table</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">write_disposition</span><span class="o">=</span><span class="s1">'WRITE_EMPTY'</span><span class="p">,</span>
+ <span class="n">allow_large_results</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">flatten_results</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">udf_config</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">use_legacy_sql</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">maximum_billing_tier</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">maximum_bytes_billed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">create_disposition</span><span class="o">=</span><span class="s1">'CREATE_IF_NEEDED'</span><span class="p">,</span>
+ <span class="n">query_params</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">schema_update_options</span><span class="o">=</span><span class="p">(),</span>
+ <span class="n">priority</span><span class="o">=</span><span class="s1">'INTERACTIVE'</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Executes a BigQuery SQL query. Optionally persists results in a BigQuery</span>
<span class="sd"> table. See here:</span>
@@ -379,23 +653,66 @@
<span class="sd"> :param write_disposition: What to do if the table already exists in</span>
<span class="sd"> BigQuery.</span>
<span class="sd"> :type write_disposition: string</span>
-<span class="sd"> :param create_disposition: Specifies whether the job is allowed to create new tables.</span>
-<span class="sd"> :type create_disposition: string</span>
<span class="sd"> :param allow_large_results: Whether to allow large results.</span>
<span class="sd"> :type allow_large_results: boolean</span>
+<span class="sd"> :param flatten_results: If true and query uses legacy SQL dialect, flattens</span>
+<span class="sd"> all nested and repeated fields in the query results. ``allowLargeResults``</span>
+<span class="sd"> must be true if this is set to false. For standard SQL queries, this</span>
+<span class="sd"> flag is ignored and results are never flattened.</span>
+<span class="sd"> :type flatten_results: boolean</span>
<span class="sd"> :param udf_config: The User Defined Function configuration for the query.</span>
<span class="sd"> See https://cloud.google.com/bigquery/user-defined-functions for details.</span>
-<span class="sd"> :type udf_config: list</span>
<span class="sd"> :param use_legacy_sql: Whether to use legacy SQL (true) or standard SQL (false).</span>
+<span class="sd"> If `None`, defaults to `self.use_legacy_sql`.</span>
<span class="sd"> :type use_legacy_sql: boolean</span>
-<span class="sd"> :param maximum_billing_tier: Positive integer that serves as a multiplier of the basic price.</span>
+<span class="sd"> :type udf_config: list</span>
+<span class="sd"> :param maximum_billing_tier: Positive integer that serves as a</span>
+<span class="sd"> multiplier of the basic price.</span>
<span class="sd"> :type maximum_billing_tier: integer</span>
+<span class="sd"> :param maximum_bytes_billed: Limits the bytes billed for this job.</span>
+<span class="sd"> Queries that will have bytes billed beyond this limit will fail</span>
+<span class="sd"> (without incurring a charge). If unspecified, this will be</span>
+<span class="sd"> set to your project default.</span>
+<span class="sd"> :type maximum_bytes_billed: float</span>
+<span class="sd"> :param create_disposition: Specifies whether the job is allowed to</span>
+<span class="sd"> create new tables.</span>
+<span class="sd"> :type create_disposition: string</span>
+<span class="sd"> :param query_params a dictionary containing query parameter types and</span>
+<span class="sd"> values, passed to BigQuery</span>
+<span class="sd"> :type query_params: dict</span>
+<span class="sd"> :param schema_update_options: Allows the schema of the desitination</span>
+<span class="sd"> table to be updated as a side effect of the query job.</span>
+<span class="sd"> :type schema_update_options: tuple</span>
+<span class="sd"> :param priority: Specifies a priority for the query.</span>
+<span class="sd"> Possible values include INTERACTIVE and BATCH.</span>
+<span class="sd"> The default value is INTERACTIVE.</span>
+<span class="sd"> :type priority: string</span>
<span class="sd"> """</span>
+
+ <span class="c1"># BigQuery also allows you to define how you want a table's schema to change</span>
+ <span class="c1"># as a side effect of a query job</span>
+ <span class="c1"># for more details:</span>
+ <span class="c1"># https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.query.schemaUpdateOptions</span>
+ <span class="n">allowed_schema_update_options</span> <span class="o">=</span> <span class="p">[</span>
+ <span class="s1">'ALLOW_FIELD_ADDITION'</span><span class="p">,</span> <span class="s2">"ALLOW_FIELD_RELAXATION"</span>
+ <span class="p">]</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="nb">set</span><span class="p">(</span><span class="n">allowed_schema_update_options</span><span class="p">)</span><span class="o">.</span><span class="n">issuperset</span><span class="p">(</span>
+ <span class="nb">set</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">)):</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+ <span class="s2">"</span><span class="si">{0}</span><span class="s2"> contains invalid schema update options. "</span>
+ <span class="s2">"Please only use one or more of the following options: </span><span class="si">{1}</span><span class="s2">"</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">,</span> <span class="n">allowed_schema_update_options</span><span class="p">))</span>
+
+ <span class="k">if</span> <span class="n">use_legacy_sql</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">use_legacy_sql</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span>
+
<span class="n">configuration</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">'query'</span><span class="p">:</span> <span class="p">{</span>
<span class="s1">'query'</span><span class="p">:</span> <span class="n">bql</span><span class="p">,</span>
<span class="s1">'useLegacySql'</span><span class="p">:</span> <span class="n">use_legacy_sql</span><span class="p">,</span>
- <span class="s1">'maximumBillingTier'</span><span class="p">:</span> <span class="n">maximum_billing_tier</span>
+ <span class="s1">'maximumBillingTier'</span><span class="p">:</span> <span class="n">maximum_billing_tier</span><span class="p">,</span>
+ <span class="s1">'maximumBytesBilled'</span><span class="p">:</span> <span class="n">maximum_bytes_billed</span><span class="p">,</span>
+ <span class="s1">'priority'</span><span class="p">:</span> <span class="n">priority</span>
<span class="p">}</span>
<span class="p">}</span>
@@ -407,9 +724,14 @@
<span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">destination_dataset_table</span><span class="p">,</span>
<span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">)</span>
<span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">({</span>
- <span class="s1">'allowLargeResults'</span><span class="p">:</span> <span class="n">allow_large_results</span><span class="p">,</span>
- <span class="s1">'writeDisposition'</span><span class="p">:</span> <span class="n">write_disposition</span><span class="p">,</span>
- <span class="s1">'createDisposition'</span><span class="p">:</span> <span class="n">create_disposition</span><span class="p">,</span>
+ <span class="s1">'allowLargeResults'</span><span class="p">:</span>
+ <span class="n">allow_large_results</span><span class="p">,</span>
+ <span class="s1">'flattenResults'</span><span class="p">:</span>
+ <span class="n">flatten_results</span><span class="p">,</span>
+ <span class="s1">'writeDisposition'</span><span class="p">:</span>
+ <span class="n">write_disposition</span><span class="p">,</span>
+ <span class="s1">'createDisposition'</span><span class="p">:</span>
+ <span class="n">create_disposition</span><span class="p">,</span>
<span class="s1">'destinationTable'</span><span class="p">:</span> <span class="p">{</span>
<span class="s1">'projectId'</span><span class="p">:</span> <span class="n">destination_project</span><span class="p">,</span>
<span class="s1">'datasetId'</span><span class="p">:</span> <span class="n">destination_dataset</span><span class="p">,</span>
@@ -419,17 +741,38 @@
<span class="k">if</span> <span class="n">udf_config</span><span class="p">:</span>
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">udf_config</span><span class="p">,</span> <span class="nb">list</span><span class="p">)</span>
<span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">({</span>
- <span class="s1">'userDefinedFunctionResources'</span><span class="p">:</span> <span class="n">udf_config</span>
+ <span class="s1">'userDefinedFunctionResources'</span><span class="p">:</span>
+ <span class="n">udf_config</span>
<span class="p">})</span>
<span class="k">if</span> <span class="n">query_params</span><span class="p">:</span>
- <span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">][</span><span class="s1">'queryParameters'</span><span class="p">]</span> <span class="o">=</span> <span class="n">query_params</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Query paramaters are not allowed when using "</span>
+ <span class="s2">"legacy SQL"</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">][</span><span class="s1">'queryParameters'</span><span class="p">]</span> <span class="o">=</span> <span class="n">query_params</span>
+
+ <span class="k">if</span> <span class="n">schema_update_options</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">write_disposition</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">"WRITE_APPEND"</span><span class="p">,</span> <span class="s2">"WRITE_TRUNCATE"</span><span class="p">]:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"schema_update_options is only "</span>
+ <span class="s2">"allowed if write_disposition is "</span>
+ <span class="s2">"'WRITE_APPEND' or 'WRITE_TRUNCATE'."</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s2">"Adding experimental "</span>
+ <span class="s2">"'schemaUpdateOptions': </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">))</span>
+ <span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">][</span>
+ <span class="s1">'schemaUpdateOptions'</span><span class="p">]</span> <span class="o">=</span> <span class="n">schema_update_options</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_with_configuration</span><span class="p">(</span><span class="n">configuration</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">run_extract</span><span class="p">(</span> <span class="c1"># noqa</span>
- <span class="bp">self</span><span class="p">,</span> <span class="n">source_project_dataset_table</span><span class="p">,</span> <span class="n">destination_cloud_storage_uris</span><span class="p">,</span>
- <span class="n">compression</span><span class="o">=</span><span class="s1">'NONE'</span><span class="p">,</span> <span class="n">export_format</span><span class="o">=</span><span class="s1">'CSV'</span><span class="p">,</span> <span class="n">field_delimiter</span><span class="o">=</span><span class="s1">','</span><span class="p">,</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">source_project_dataset_table</span><span class="p">,</span>
+ <span class="n">destination_cloud_storage_uris</span><span class="p">,</span>
+ <span class="n">compression</span><span class="o">=</span><span class="s1">'NONE'</span><span class="p">,</span>
+ <span class="n">export_format</span><span class="o">=</span><span class="s1">'CSV'</span><span class="p">,</span>
+ <span class="n">field_delimiter</span><span class="o">=</span><span class="s1">','</span><span class="p">,</span>
<span class="n">print_header</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Executes a BigQuery extract command to copy data from BigQuery to</span>
@@ -512,10 +855,10 @@
<span class="sd"> :param create_disposition: The create disposition if the table doesn't exist.</span>
<span class="sd"> :type create_disposition: string</span>
<span class="sd"> """</span>
- <span class="n">source_project_dataset_tables</span> <span class="o">=</span> <span class="p">(</span>
- <span class="p">[</span><span class="n">source_project_dataset_tables</span><span class="p">]</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">source_project_dataset_tables</span><span class="p">,</span> <span class="nb">list</span><span class="p">)</span>
- <span class="k">else</span> <span class="n">source_project_dataset_tables</span><span class="p">)</span>
+ <span class="n">source_project_dataset_tables</span> <span class="o">=</span> <span class="p">([</span>
+ <span class="n">source_project_dataset_tables</span>
+ <span class="p">]</span> <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">source_project_dataset_tables</span><span class="p">,</span> <span class="nb">list</span><span class="p">)</span> <span class="k">else</span>
+ <span class="n">source_project_dataset_tables</span><span class="p">)</span>
<span class="n">source_project_dataset_tables_fixup</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">source_project_dataset_table</span> <span class="ow">in</span> <span class="n">source_project_dataset_tables</span><span class="p">:</span>
@@ -524,9 +867,12 @@
<span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
<span class="n">var_name</span><span class="o">=</span><span class="s1">'source_project_dataset_table'</span><span class="p">)</span>
<span class="n">source_project_dataset_tables_fixup</span><span class="o">.</span><span class="n">append</span><span class="p">({</span>
- <span class="s1">'projectId'</span><span class="p">:</span> <span class="n">source_project</span><span class="p">,</span>
- <span class="s1">'datasetId'</span><span class="p">:</span> <span class="n">source_dataset</span><span class="p">,</span>
- <span class="s1">'tableId'</span><span class="p">:</span> <span class="n">source_table</span>
+ <span class="s1">'projectId'</span><span class="p">:</span>
+ <span class="n">source_project</span><span class="p">,</span>
+ <span class="s1">'datasetId'</span><span class="p">:</span>
+ <span class="n">source_dataset</span><span class="p">,</span>
+ <span class="s1">'tableId'</span><span class="p">:</span>
+ <span class="n">source_table</span>
<span class="p">})</span>
<span class="n">destination_project</span><span class="p">,</span> <span class="n">destination_dataset</span><span class="p">,</span> <span class="n">destination_table</span> <span class="o">=</span> \
@@ -549,7 +895,8 @@
<span class="k">def</span> <span class="nf">run_load</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
<span class="n">destination_project_dataset_table</span><span class="p">,</span>
- <span class="n">schema_fields</span><span class="p">,</span> <span class="n">source_uris</span><span class="p">,</span>
+ <span class="n">schema_fields</span><span class="p">,</span>
+ <span class="n">source_uris</span><span class="p">,</span>
<span class="n">source_format</span><span class="o">=</span><span class="s1">'CSV'</span><span class="p">,</span>
<span class="n">create_disposition</span><span class="o">=</span><span class="s1">'CREATE_IF_NEEDED'</span><span class="p">,</span>
<span class="n">skip_leading_rows</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
@@ -557,10 +904,12 @@
<span class="n">field_delimiter</span><span class="o">=</span><span class="s1">','</span><span class="p">,</span>
<span class="n">max_bad_records</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
<span class="n">quote_character</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">ignore_unknown_values</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">allow_quoted_newlines</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">allow_jagged_rows</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">schema_update_options</span><span class="o">=</span><span class="p">(),</span>
- <span class="n">src_fmt_configs</span><span class="o">=</span><span class="p">{}):</span>
+ <span class="n">src_fmt_configs</span><span class="o">=</span><span class="p">{},</span>
+ <span class="n">time_partitioning</span><span class="o">=</span><span class="p">{}):</span>
<span class="sd">"""</span>
<span class="sd"> Executes a BigQuery load command to load data from Google Cloud Storage</span>
<span class="sd"> to BigQuery. See here:</span>
@@ -570,9 +919,11 @@
<span class="sd"> For more details about these parameters.</span>
<span class="sd"> :param destination_project_dataset_table:</span>
-<span class="sd"> The dotted (<project>.|<project>:)<dataset>.<table> BigQuery table to load</span>
-<span class="sd"> data into. If <project> is not included, project will be the project defined</span>
-<span class="sd"> in the connection json.</span>
+<span class="sd"> The dotted (<project>.|<project>:)<dataset>.<table>($<partition>) BigQuery</span>
+<span class="sd"> table to load data into. If <project> is not included, project will be the</span>
+<span class="sd"> project defined in the connection json. If a partition is specified the</span>
+<span class="sd"> operator will automatically append the data, create a new partition or create</span>
+<span class="sd"> a new DAY partitioned table.</span>
<span class="sd"> :type destination_project_dataset_table: string</span>
<span class="sd"> :param schema_fields: The schema field list as defined here:</span>
<span class="sd"> https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load</span>
@@ -594,20 +945,34 @@
<span class="sd"> :param max_bad_records: The maximum number of bad records that BigQuery can</span>
<span class="sd"> ignore when running the job.</span>
<span class="sd"> :type max_bad_records: int</span>
-<span class="sd"> :param quote_character: The value that is used to quote data sections in a CSV file.</span>
+<span class="sd"> :param quote_character: The value that is used to quote data sections in a CSV</span>
+<span class="sd"> file.</span>
<span class="sd"> :type quote_character: string</span>
-<span class="sd"> :param allow_quoted_newlines: Whether to allow quoted newlines (true) or not (false).</span>
+<span class="sd"> :param ignore_unknown_values: [Optional] Indicates if BigQuery should allow</span>
+<span class="sd"> extra values that are not represented in the table schema.</span>
+<span class="sd"> If true, the extra values are ignored. If false, records with extra columns</span>
+<span class="sd"> are treated as bad records, and if there are too many bad records, an</span>
+<span class="sd"> invalid error is returned in the job result.</span>
+<span class="sd"> :type ignore_unknown_values: bool</span>
+<span class="sd"> :param allow_quoted_newlines: Whether to allow quoted newlines (true) or not</span>
+<span class="sd"> (false).</span>
<span class="sd"> :type allow_quoted_newlines: boolean</span>
<span class="sd"> :param allow_jagged_rows: Accept rows that are missing trailing optional columns.</span>
-<span class="sd"> The missing values are treated as nulls. If false, records with missing trailing columns</span>
-<span class="sd"> are treated as bad records, and if there are too many bad records, an invalid error is</span>
-<span class="sd"> returned in the job result. Only applicable when soure_format is CSV.</span>
+<span class="sd"> The missing values are treated as nulls. If false, records with missing</span>
+<span class="sd"> trailing columns are treated as bad records, and if there are too many bad</span>
+<span class="sd"> records, an invalid error is returned in the job result. Only applicable when</span>
+<span class="sd"> soure_format is CSV.</span>
<span class="sd"> :type allow_jagged_rows: bool</span>
<span class="sd"> :param schema_update_options: Allows the schema of the desitination</span>
<span class="sd"> table to be updated as a side effect of the load job.</span>
-<span class="sd"> :type schema_update_options: list</span>
+<span class="sd"> :type schema_update_options: tuple</span>
<span class="sd"> :param src_fmt_configs: configure optional fields specific to the source format</span>
<span class="sd"> :type src_fmt_configs: dict</span>
+<span class="sd"> :param time_partitioning: configure optional time partitioning fields i.e.</span>
+<span class="sd"> partition by field, type and</span>
+<span class="sd"> expiration as per API specifications. Note that 'field' is not available in</span>
+<span class="sd"> concurrency with dataset.table$partition.</span>
+<span class="sd"> :type time_partitioning: dict</span>
<span class="sd"> """</span>
<span class="c1"># bigquery only allows certain source formats</span>
@@ -616,26 +981,28 @@
<span class="c1"># Refer to this link for more details:</span>
<span class="c1"># https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.query.tableDefinitions.(key).sourceFormat</span>
<span class="n">source_format</span> <span class="o">=</span> <span class="n">source_format</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
- <span class="n">allowed_formats</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"CSV"</span><span class="p">,</span> <span class="s2">"NEWLINE_DELIMITED_JSON"</span><span class="p">,</span> <span class="s2">"AVRO"</span><span class="p">,</span> <span class="s2">"GOOGLE_SHEETS"</span><span class="p">,</span> <span class="s2">"DATASTORE_BACKUP"</span><span class="p">]</span>
+ <span class="n">allowed_formats</span> <span class="o">=</span> <span class="p">[</span>
+ <span class="s2">"CSV"</span><span class="p">,</span> <span class="s2">"NEWLINE_DELIMITED_JSON"</span><span class="p">,</span> <span class="s2">"AVRO"</span><span class="p">,</span> <span class="s2">"GOOGLE_SHEETS"</span><span class="p">,</span>
+ <span class="s2">"DATASTORE_BACKUP"</span><span class="p">,</span> <span class="s2">"PARQUET"</span>
+ <span class="p">]</span>
<span class="k">if</span> <span class="n">source_format</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">allowed_formats</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"</span><span class="si">{0}</span><span class="s2"> is not a valid source format. "</span>
- <span class="s2">"Please use one of the following types: </span><span class="si">{1}</span><span class="s2">"</span>
- <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">source_format</span><span class="p">,</span> <span class="n">allowed_formats</span><span class="p">))</span>
+ <span class="s2">"Please use one of the following types: </span><span class="si">{1}</span><span class="s2">"</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">source_format</span><span class="p">,</span> <span class="n">allowed_formats</span><span class="p">))</span>
<span class="c1"># bigquery also allows you to define how you want a table's schema to change</span>
<span class="c1"># as a side effect of a load</span>
<span class="c1"># for more details:</span>
- <span class="c1"># https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load.schemaUpdateOptions</span>
+ <span class="c1"># https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load.schemaUpdateOptions</span>
<span class="n">allowed_schema_update_options</span> <span class="o">=</span> <span class="p">[</span>
- <span class="s1">'ALLOW_FIELD_ADDITION'</span><span class="p">,</span>
- <span class="s2">"ALLOW_FIELD_RELAXATION"</span>
+ <span class="s1">'ALLOW_FIELD_ADDITION'</span><span class="p">,</span> <span class="s2">"ALLOW_FIELD_RELAXATION"</span>
<span class="p">]</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="nb">set</span><span class="p">(</span><span class="n">allowed_schema_update_options</span><span class="p">)</span><span class="o">.</span><span class="n">issuperset</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">)):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="nb">set</span><span class="p">(</span><span class="n">allowed_schema_update_options</span><span class="p">)</span><span class="o">.</span><span class="n">issuperset</span><span class="p">(</span>
+ <span class="nb">set</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">)):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> contains invalid schema update options. "</span>
<span class="s2">"Please only use one or more of the following options: </span><span class="si">{1}</span><span class="s2">"</span>
- <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">,</span> <span class="n">allowed_schema_update_options</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">,</span> <span class="n">allowed_schema_update_options</span><span class="p">))</span>
<span class="n">destination_project</span><span class="p">,</span> <span class="n">destination_dataset</span><span class="p">,</span> <span class="n">destination_table</span> <span class="o">=</span> \
<span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">destination_project_dataset_table</span><span class="p">,</span>
@@ -653,26 +1020,40 @@
<span class="s1">'sourceFormat'</span><span class="p">:</span> <span class="n">source_format</span><span class="p">,</span>
<span class="s1">'sourceUris'</span><span class="p">:</span> <span class="n">source_uris</span><span class="p">,</span>
<span class="s1">'writeDisposition'</span><span class="p">:</span> <span class="n">write_disposition</span><span class="p">,</span>
+ <span class="s1">'ignoreUnknownValues'</span><span class="p">:</span> <span class="n">ignore_unknown_values</span>
<span class="p">}</span>
<span class="p">}</span>
+
+ <span class="c1"># if it is a partitioned table ($ is in the table name) add partition load option</span>
+ <span class="k">if</span> <span class="s1">'$'</span> <span class="ow">in</span> <span class="n">destination_project_dataset_table</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">time_partitioning</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'field'</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s2">"Cannot specify field partition and partition name "</span>
+ <span class="s2">"(dataset.table$partition) at the same time"</span>
+ <span class="p">)</span>
+ <span class="n">configuration</span><span class="p">[</span><span class="s1">'load'</span><span class="p">][</span><span class="s1">'timePartitioning'</span><span class="p">]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n
<TRUNCATED>
[47/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/discord_webhook_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/discord_webhook_hook.html b/_modules/airflow/contrib/hooks/discord_webhook_hook.html
new file mode 100644
index 0000000..1115b8c
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/discord_webhook_hook.html
@@ -0,0 +1,375 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.discord_webhook_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.discord_webhook_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.discord_webhook_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+<span class="kn">import</span> <span class="nn">json</span>
+<span class="kn">import</span> <span class="nn">re</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.http_hook</span> <span class="k">import</span> <span class="n">HttpHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+
+
+<div class="viewcode-block" id="DiscordWebhookHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.discord_webhook_hook.DiscordWebhookHook">[docs]</a><span class="k">class</span> <span class="nc">DiscordWebhookHook</span><span class="p">(</span><span class="n">HttpHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> This hook allows you to post messages to Discord using incoming webhooks.</span>
+<span class="sd"> Takes a Discord connection ID with a default relative webhook endpoint. The</span>
+<span class="sd"> default endpoint can be overridden using the webhook_endpoint parameter</span>
+<span class="sd"> (https://discordapp.com/developers/docs/resources/webhook).</span>
+
+<span class="sd"> Each Discord webhook can be pre-configured to use a specific username and</span>
+<span class="sd"> avatar_url. You can override these defaults in this hook.</span>
+
+<span class="sd"> :param http_conn_id: Http connection ID with host as "https://discord.com/api/" and</span>
+<span class="sd"> default webhook endpoint in the extra field in the form of</span>
+<span class="sd"> {"webhook_endpoint": "webhooks/{webhook.id}/{webhook.token}"}</span>
+<span class="sd"> :type http_conn_id: str</span>
+<span class="sd"> :param webhook_endpoint: Discord webhook endpoint in the form of</span>
+<span class="sd"> "webhooks/{webhook.id}/{webhook.token}"</span>
+<span class="sd"> :type webhook_endpoint: str</span>
+<span class="sd"> :param message: The message you want to send to your Discord channel</span>
+<span class="sd"> (max 2000 characters)</span>
+<span class="sd"> :type message: str</span>
+<span class="sd"> :param username: Override the default username of the webhook</span>
+<span class="sd"> :type username: str</span>
+<span class="sd"> :param avatar_url: Override the default avatar of the webhook</span>
+<span class="sd"> :type avatar_url: str</span>
+<span class="sd"> :param tts: Is a text-to-speech message</span>
+<span class="sd"> :type tts: bool</span>
+<span class="sd"> :param proxy: Proxy to use to make the Discord webhook call</span>
+<span class="sd"> :type proxy: str</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">http_conn_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">webhook_endpoint</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">message</span><span class="o">=</span><span class="s2">""</span><span class="p">,</span>
+ <span class="n">username</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">avatar_url</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">tts</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">proxy</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">DiscordWebhookHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">http_conn_id</span> <span class="o">=</span> <span class="n">http_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">webhook_endpoint</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_webhook_endpoint</span><span class="p">(</span><span class="n">http_conn_id</span><span class="p">,</span> <span class="n">webhook_endpoint</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">message</span> <span class="o">=</span> <span class="n">message</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="o">=</span> <span class="n">username</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">avatar_url</span> <span class="o">=</span> <span class="n">avatar_url</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">tts</span> <span class="o">=</span> <span class="n">tts</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">proxy</span> <span class="o">=</span> <span class="n">proxy</span>
+
+ <span class="k">def</span> <span class="nf">_get_webhook_endpoint</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">http_conn_id</span><span class="p">,</span> <span class="n">webhook_endpoint</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Given a Discord http_conn_id, return the default webhook endpoint or override if a</span>
+<span class="sd"> webhook_endpoint is manually supplied.</span>
+
+<span class="sd"> :param http_conn_id: The provided connection ID</span>
+<span class="sd"> :param webhook_endpoint: The manually provided webhook endpoint</span>
+<span class="sd"> :return: Webhook endpoint (str) to use</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">webhook_endpoint</span><span class="p">:</span>
+ <span class="n">endpoint</span> <span class="o">=</span> <span class="n">webhook_endpoint</span>
+ <span class="k">elif</span> <span class="n">http_conn_id</span><span class="p">:</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">http_conn_id</span><span class="p">)</span>
+ <span class="n">extra</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span>
+ <span class="n">endpoint</span> <span class="o">=</span> <span class="n">extra</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'webhook_endpoint'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Cannot get webhook endpoint: No valid Discord '</span>
+ <span class="s1">'webhook endpoint or http_conn_id supplied.'</span><span class="p">)</span>
+
+ <span class="c1"># make sure endpoint matches the expected Discord webhook format</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="s1">'^webhooks/[0-9]+/[a-zA-Z0-9_-]+$'</span><span class="p">,</span> <span class="n">endpoint</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Expected Discord webhook endpoint in the form '</span>
+ <span class="s1">'of "webhooks/</span><span class="si">{webhook.id}</span><span class="s1">/</span><span class="si">{webhook.token}</span><span class="s1">".'</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">endpoint</span>
+
+ <span class="k">def</span> <span class="nf">_build_discord_payload</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Construct the Discord JSON payload. All relevant parameters are combined here</span>
+<span class="sd"> to a valid Discord JSON payload.</span>
+
+<span class="sd"> :return: Discord payload (str) to send</span>
+<span class="sd"> """</span>
+ <span class="n">payload</span> <span class="o">=</span> <span class="p">{}</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">:</span>
+ <span class="n">payload</span><span class="p">[</span><span class="s1">'username'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">username</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">avatar_url</span><span class="p">:</span>
+ <span class="n">payload</span><span class="p">[</span><span class="s1">'avatar_url'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">avatar_url</span>
+
+ <span class="n">payload</span><span class="p">[</span><span class="s1">'tts'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">tts</span>
+
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">message</span><span class="p">)</span> <span class="o"><=</span> <span class="mi">2000</span><span class="p">:</span>
+ <span class="n">payload</span><span class="p">[</span><span class="s1">'content'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">message</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Discord message length must be 2000 or fewer '</span>
+ <span class="s1">'characters.'</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
+
+<div class="viewcode-block" id="DiscordWebhookHook.execute"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.discord_webhook_hook.DiscordWebhookHook.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Execute the Discord webhook call</span>
+<span class="sd"> """</span>
+ <span class="n">proxies</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">proxy</span><span class="p">:</span>
+ <span class="c1"># we only need https proxy for Discord</span>
+ <span class="n">proxies</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'https'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">proxy</span><span class="p">}</span>
+
+ <span class="n">discord_payload</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_discord_payload</span><span class="p">()</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">endpoint</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">webhook_endpoint</span><span class="p">,</span>
+ <span class="n">data</span><span class="o">=</span><span class="n">discord_payload</span><span class="p">,</span>
+ <span class="n">headers</span><span class="o">=</span><span class="p">{</span><span class="s1">'Content-type'</span><span class="p">:</span> <span class="s1">'application/json'</span><span class="p">},</span>
+ <span class="n">extra_options</span><span class="o">=</span><span class="p">{</span><span class="s1">'proxies'</span><span class="p">:</span> <span class="n">proxies</span><span class="p">})</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/emr_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/emr_hook.html b/_modules/airflow/contrib/hooks/emr_hook.html
index cc158a2..8a5a5c7 100644
--- a/_modules/airflow/contrib/hooks/emr_hook.html
+++ b/_modules/airflow/contrib/hooks/emr_hook.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,22 @@
<h1>Source code for airflow.contrib.hooks.emr_hook</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.aws_hook</span> <span class="k">import</span> <span class="n">AwsHook</span>
@@ -198,7 +205,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_client_type</span><span class="p">(</span><span class="s1">'emr'</span><span class="p">)</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span>
- <span class="k">def</span> <span class="nf">create_job_flow</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">job_flow_overrides</span><span class="p">):</span>
+<div class="viewcode-block" id="EmrHook.create_job_flow"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.emr_hook.EmrHook.create_job_flow">[docs]</a> <span class="k">def</span> <span class="nf">create_job_flow</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">job_flow_overrides</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Creates a job flow using the config from the EMR connection.</span>
<span class="sd"> Keys of the json extra hash may have the arguments of the boto3 run_job_flow method.</span>
@@ -228,7 +235,7 @@
<span class="n">Tags</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Tags'</span><span class="p">)</span>
<span class="p">)</span>
- <span class="k">return</span> <span class="n">response</span></div>
+ <span class="k">return</span> <span class="n">response</span></div></div>
</pre></div>
</div>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/fs_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/fs_hook.html b/_modules/airflow/contrib/hooks/fs_hook.html
new file mode 100644
index 0000000..984dd40
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/fs_hook.html
@@ -0,0 +1,281 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.fs_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.fs_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.fs_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+
+
+<div class="viewcode-block" id="FSHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.fs_hook.FSHook">[docs]</a><span class="k">class</span> <span class="nc">FSHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+ <span class="sd">'''</span>
+<span class="sd"> Allows for interaction with an file server.</span>
+
+<span class="sd"> Connection should have a name and a path specified under extra:</span>
+
+<span class="sd"> example:</span>
+<span class="sd"> Conn Id: fs_test</span>
+<span class="sd"> Conn Type: File (path)</span>
+<span class="sd"> Host, Shchema, Login, Password, Port: empty</span>
+<span class="sd"> Extra: {"path": "/tmp"}</span>
+<span class="sd"> '''</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn_id</span><span class="o">=</span><span class="s1">'fs_default'</span><span class="p">):</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">conn_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">basepath</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'path'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="n">conn</span>
+
+ <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">pass</span>
+
+ <span class="k">def</span> <span class="nf">get_path</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">basepath</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/ftp_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/ftp_hook.html b/_modules/airflow/contrib/hooks/ftp_hook.html
new file mode 100644
index 0000000..07b434f
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/ftp_hook.html
@@ -0,0 +1,494 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.ftp_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.ftp_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.ftp_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+
+<span class="kn">import</span> <span class="nn">datetime</span>
+<span class="kn">import</span> <span class="nn">ftplib</span>
+<span class="kn">import</span> <span class="nn">os.path</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="k">import</span> <span class="n">basestring</span>
+
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+
+<span class="k">def</span> <span class="nf">mlsd</span><span class="p">(</span><span class="n">conn</span><span class="p">,</span> <span class="n">path</span><span class="o">=</span><span class="s2">""</span><span class="p">,</span> <span class="n">facts</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> BACKPORT FROM PYTHON3 FTPLIB.</span>
+
+<span class="sd"> List a directory in a standardized format by using MLSD</span>
+<span class="sd"> command (RFC-3659). If path is omitted the current directory</span>
+<span class="sd"> is assumed. "facts" is a list of strings representing the type</span>
+<span class="sd"> of information desired (e.g. ["type", "size", "perm"]).</span>
+
+<span class="sd"> Return a generator object yielding a tuple of two elements</span>
+<span class="sd"> for every file found in path.</span>
+<span class="sd"> First element is the file name, the second one is a dictionary</span>
+<span class="sd"> including a variable number of "facts" depending on the server</span>
+<span class="sd"> and whether "facts" argument has been provided.</span>
+<span class="sd"> """</span>
+ <span class="n">facts</span> <span class="o">=</span> <span class="n">facts</span> <span class="ow">or</span> <span class="p">[]</span>
+ <span class="k">if</span> <span class="n">facts</span><span class="p">:</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">sendcmd</span><span class="p">(</span><span class="s2">"OPTS MLST "</span> <span class="o">+</span> <span class="s2">";"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">facts</span><span class="p">)</span> <span class="o">+</span> <span class="s2">";"</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">path</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">=</span> <span class="s2">"MLSD </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">path</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">=</span> <span class="s2">"MLSD"</span>
+ <span class="n">lines</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">retrlines</span><span class="p">(</span><span class="n">cmd</span><span class="p">,</span> <span class="n">lines</span><span class="o">.</span><span class="n">append</span><span class="p">)</span>
+ <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">lines</span><span class="p">:</span>
+ <span class="n">facts_found</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">name</span> <span class="o">=</span> <span class="n">line</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="n">ftplib</span><span class="o">.</span><span class="n">CRLF</span><span class="p">)</span><span class="o">.</span><span class="n">partition</span><span class="p">(</span><span class="s1">' '</span><span class="p">)</span>
+ <span class="n">entry</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="k">for</span> <span class="n">fact</span> <span class="ow">in</span> <span class="n">facts_found</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">";"</span><span class="p">):</span>
+ <span class="n">key</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">value</span> <span class="o">=</span> <span class="n">fact</span><span class="o">.</span><span class="n">partition</span><span class="p">(</span><span class="s2">"="</span><span class="p">)</span>
+ <span class="n">entry</span><span class="p">[</span><span class="n">key</span><span class="o">.</span><span class="n">lower</span><span class="p">()]</span> <span class="o">=</span> <span class="n">value</span>
+ <span class="k">yield</span> <span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">entry</span><span class="p">)</span>
+
+
+<div class="viewcode-block" id="FTPHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPHook">[docs]</a><span class="k">class</span> <span class="nc">FTPHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with FTP.</span>
+
+<span class="sd"> Errors that may occur throughout but should be handled</span>
+<span class="sd"> downstream.</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ftp_conn_id</span><span class="o">=</span><span class="s1">'ftp_default'</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">ftp_conn_id</span> <span class="o">=</span> <span class="n">ftp_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="kc">None</span>
+
+ <span class="k">def</span> <span class="nf">__enter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span>
+
+ <span class="k">def</span> <span class="nf">__exit__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">exc_type</span><span class="p">,</span> <span class="n">exc_val</span><span class="p">,</span> <span class="n">exc_tb</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">close_conn</span><span class="p">()</span>
+
+<div class="viewcode-block" id="FTPHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns a FTP connection object</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">params</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ftp_conn_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="n">ftplib</span><span class="o">.</span><span class="n">FTP</span><span class="p">(</span><span class="n">params</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> <span class="n">params</span><span class="o">.</span><span class="n">login</span><span class="p">,</span> <span class="n">params</span><span class="o">.</span><span class="n">password</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span></div>
+
+<div class="viewcode-block" id="FTPHook.close_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPHook.close_conn">[docs]</a> <span class="k">def</span> <span class="nf">close_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Closes the connection. An error will occur if the</span>
+<span class="sd"> connection wasn't ever opened.</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">quit</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="kc">None</span></div>
+
+<div class="viewcode-block" id="FTPHook.describe_directory"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPHook.describe_directory">[docs]</a> <span class="k">def</span> <span class="nf">describe_directory</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns a dictionary of {filename: {attributes}} for all files</span>
+<span class="sd"> on the remote system (where the MLSD command is supported).</span>
+
+<span class="sd"> :param path: full path to the remote directory</span>
+<span class="sd"> :type path: str</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">cwd</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="c1"># only works in Python 3</span>
+ <span class="n">files</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">mlsd</span><span class="p">())</span>
+ <span class="k">except</span> <span class="ne">AttributeError</span><span class="p">:</span>
+ <span class="n">files</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">mlsd</span><span class="p">(</span><span class="n">conn</span><span class="p">))</span>
+ <span class="k">return</span> <span class="n">files</span></div>
+
+<div class="viewcode-block" id="FTPHook.list_directory"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPHook.list_directory">[docs]</a> <span class="k">def</span> <span class="nf">list_directory</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">nlst</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns a list of files on the remote system.</span>
+
+<span class="sd"> :param path: full path to the remote directory to list</span>
+<span class="sd"> :type path: str</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">cwd</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
+
+ <span class="n">files</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">nlst</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">files</span></div>
+
+<div class="viewcode-block" id="FTPHook.create_directory"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPHook.create_directory">[docs]</a> <span class="k">def</span> <span class="nf">create_directory</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Creates a directory on the remote system.</span>
+
+<span class="sd"> :param path: full path to the remote directory to create</span>
+<span class="sd"> :type path: str</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">mkd</span><span class="p">(</span><span class="n">path</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="FTPHook.delete_directory"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPHook.delete_directory">[docs]</a> <span class="k">def</span> <span class="nf">delete_directory</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Deletes a directory on the remote system.</span>
+
+<span class="sd"> :param path: full path to the remote directory to delete</span>
+<span class="sd"> :type path: str</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">rmd</span><span class="p">(</span><span class="n">path</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="FTPHook.retrieve_file"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPHook.retrieve_file">[docs]</a> <span class="k">def</span> <span class="nf">retrieve_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">remote_full_path</span><span class="p">,</span> <span class="n">local_full_path_or_buffer</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Transfers the remote file to a local location.</span>
+
+<span class="sd"> If local_full_path_or_buffer is a string path, the file will be put</span>
+<span class="sd"> at that location; if it is a file-like buffer, the file will</span>
+<span class="sd"> be written to the buffer but not closed.</span>
+
+<span class="sd"> :param remote_full_path: full path to the remote file</span>
+<span class="sd"> :type remote_full_path: str</span>
+<span class="sd"> :param local_full_path_or_buffer: full path to the local file or a</span>
+<span class="sd"> file-like buffer</span>
+<span class="sd"> :type local_full_path_or_buffer: str or file-like buffer</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+ <span class="n">is_path</span> <span class="o">=</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">local_full_path_or_buffer</span><span class="p">,</span> <span class="n">basestring</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">is_path</span><span class="p">:</span>
+ <span class="n">output_handle</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">local_full_path_or_buffer</span><span class="p">,</span> <span class="s1">'wb'</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">output_handle</span> <span class="o">=</span> <span class="n">local_full_path_or_buffer</span>
+
+ <span class="n">remote_path</span><span class="p">,</span> <span class="n">remote_file_name</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">remote_full_path</span><span class="p">)</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">cwd</span><span class="p">(</span><span class="n">remote_path</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Retrieving file from FTP: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">remote_full_path</span><span class="p">)</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">retrbinary</span><span class="p">(</span><span class="s1">'RETR </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">remote_file_name</span><span class="p">,</span> <span class="n">output_handle</span><span class="o">.</span><span class="n">write</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Finished retrieving file from FTP: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">remote_full_path</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">is_path</span><span class="p">:</span>
+ <span class="n">output_handle</span><span class="o">.</span><span class="n">close</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="FTPHook.store_file"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPHook.store_file">[docs]</a> <span class="k">def</span> <span class="nf">store_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">remote_full_path</span><span class="p">,</span> <span class="n">local_full_path_or_buffer</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Transfers a local file to the remote location.</span>
+
+<span class="sd"> If local_full_path_or_buffer is a string path, the file will be read</span>
+<span class="sd"> from that location; if it is a file-like buffer, the file will</span>
+<span class="sd"> be read from the buffer but not closed.</span>
+
+<span class="sd"> :param remote_full_path: full path to the remote file</span>
+<span class="sd"> :type remote_full_path: str</span>
+<span class="sd"> :param local_full_path_or_buffer: full path to the local file or a</span>
+<span class="sd"> file-like buffer</span>
+<span class="sd"> :type local_full_path_or_buffer: str or file-like buffer</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+ <span class="n">is_path</span> <span class="o">=</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">local_full_path_or_buffer</span><span class="p">,</span> <span class="n">basestring</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">is_path</span><span class="p">:</span>
+ <span class="n">input_handle</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">local_full_path_or_buffer</span><span class="p">,</span> <span class="s1">'rb'</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">input_handle</span> <span class="o">=</span> <span class="n">local_full_path_or_buffer</span>
+ <span class="n">remote_path</span><span class="p">,</span> <span class="n">remote_file_name</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">remote_full_path</span><span class="p">)</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">cwd</span><span class="p">(</span><span class="n">remote_path</span><span class="p">)</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">storbinary</span><span class="p">(</span><span class="s1">'STOR </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">remote_file_name</span><span class="p">,</span> <span class="n">input_handle</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">is_path</span><span class="p">:</span>
+ <span class="n">input_handle</span><span class="o">.</span><span class="n">close</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="FTPHook.delete_file"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPHook.delete_file">[docs]</a> <span class="k">def</span> <span class="nf">delete_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Removes a file on the FTP Server.</span>
+
+<span class="sd"> :param path: full path to the remote file</span>
+<span class="sd"> :type path: str</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">path</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="FTPHook.rename"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPHook.rename">[docs]</a> <span class="k">def</span> <span class="nf">rename</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">from_name</span><span class="p">,</span> <span class="n">to_name</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Rename a file.</span>
+
+<span class="sd"> :param from_name: rename file from name</span>
+<span class="sd"> :param to_name: rename file to name</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">conn</span><span class="o">.</span><span class="n">rename</span><span class="p">(</span><span class="n">from_name</span><span class="p">,</span> <span class="n">to_name</span><span class="p">)</span></div>
+
+ <span class="k">def</span> <span class="nf">get_mod_time</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">ftp_mdtm</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">sendcmd</span><span class="p">(</span><span class="s1">'MDTM '</span> <span class="o">+</span> <span class="n">path</span><span class="p">)</span>
+ <span class="n">time_val</span> <span class="o">=</span> <span class="n">ftp_mdtm</span><span class="p">[</span><span class="mi">4</span><span class="p">:]</span>
+ <span class="c1"># time_val optionally has microseconds</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">time_val</span><span class="p">,</span> <span class="s2">"%Y%m</span><span class="si">%d</span><span class="s2">%H%M%S.</span><span class="si">%f</span><span class="s2">"</span><span class="p">)</span>
+ <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">time_val</span><span class="p">,</span> <span class="s1">'%Y%m</span><span class="si">%d</span><span class="s1">%H%M%S'</span><span class="p">)</span></div>
+
+
+<div class="viewcode-block" id="FTPSHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPSHook">[docs]</a><span class="k">class</span> <span class="nc">FTPSHook</span><span class="p">(</span><span class="n">FTPHook</span><span class="p">):</span>
+
+<div class="viewcode-block" id="FTPSHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPSHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns a FTPS connection object.</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">params</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ftp_conn_id</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">params</span><span class="o">.</span><span class="n">port</span><span class="p">:</span>
+ <span class="n">ftplib</span><span class="o">.</span><span class="n">FTP_TLS</span><span class="o">.</span><span class="n">port</span><span class="o">=</span><span class="n">params</span><span class="o">.</span><span class="n">port</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="n">ftplib</span><span class="o">.</span><span class="n">FTP_TLS</span><span class="p">(</span>
+ <span class="n">params</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> <span class="n">params</span><span class="o">.</span><span class="n">login</span><span class="p">,</span> <span class="n">params</span><span class="o">.</span><span class="n">password</span>
+ <span class="p">)</span>
+
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[36/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/awsbatch_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/awsbatch_operator.html b/_modules/airflow/contrib/operators/awsbatch_operator.html
new file mode 100644
index 0000000..33e1e8b
--- /dev/null
+++ b/_modules/airflow/contrib/operators/awsbatch_operator.html
@@ -0,0 +1,403 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.awsbatch_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.awsbatch_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.awsbatch_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+<span class="kn">import</span> <span class="nn">sys</span>
+
+<span class="kn">from</span> <span class="nn">math</span> <span class="k">import</span> <span class="nb">pow</span>
+<span class="kn">from</span> <span class="nn">time</span> <span class="k">import</span> <span class="n">sleep</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.aws_hook</span> <span class="k">import</span> <span class="n">AwsHook</span>
+
+
+<div class="viewcode-block" id="AWSBatchOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.awsbatch_operator.AWSBatchOperator">[docs]</a><span class="k">class</span> <span class="nc">AWSBatchOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Execute a job on AWS Batch Service</span>
+
+<span class="sd"> :param job_name: the name for the job that will run on AWS Batch</span>
+<span class="sd"> :type job_name: str</span>
+<span class="sd"> :param job_definition: the job definition name on AWS Batch</span>
+<span class="sd"> :type job_definition: str</span>
+<span class="sd"> :param queue: the queue name on AWS Batch</span>
+<span class="sd"> :type queue: str</span>
+<span class="sd"> :param: overrides: the same parameter that boto3 will receive on containerOverrides:</span>
+<span class="sd"> http://boto3.readthedocs.io/en/latest/reference/services/batch.html#submit_job</span>
+<span class="sd"> :type: overrides: dict</span>
+<span class="sd"> :param max_retries: exponential backoff retries while waiter is not merged</span>
+<span class="sd"> :type max_retries: int</span>
+<span class="sd"> :param aws_conn_id: connection id of AWS credentials / region name. If None,</span>
+<span class="sd"> credential boto3 strategy will be used (http://boto3.readthedocs.io/en/latest/guide/configuration.html).</span>
+<span class="sd"> :type aws_conn_id: str</span>
+<span class="sd"> :param region_name: region name to use in AWS Hook. Override the region_name in connection (if provided)</span>
+<span class="sd"> """</span>
+
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#c3dae0'</span>
+ <span class="n">client</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="n">arn</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'overrides'</span><span class="p">,)</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">job_name</span><span class="p">,</span> <span class="n">job_definition</span><span class="p">,</span> <span class="n">queue</span><span class="p">,</span> <span class="n">overrides</span><span class="p">,</span> <span class="n">max_retries</span><span class="o">=</span><span class="mi">288</span><span class="p">,</span>
+ <span class="n">aws_conn_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">AWSBatchOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">job_name</span> <span class="o">=</span> <span class="n">job_name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span> <span class="o">=</span> <span class="n">aws_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">region_name</span> <span class="o">=</span> <span class="n">region_name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job_definition</span> <span class="o">=</span> <span class="n">job_definition</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">queue</span> <span class="o">=</span> <span class="n">queue</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">overrides</span> <span class="o">=</span> <span class="n">overrides</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">max_retries</span> <span class="o">=</span> <span class="n">max_retries</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">jobId</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">jobName</span> <span class="o">=</span> <span class="kc">None</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'Running AWS Batch Job - Job definition: </span><span class="si">%s</span><span class="s1"> - on queue </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job_definition</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">queue</span>
+ <span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'AWSBatchOperator overrides: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">overrides</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">get_client_type</span><span class="p">(</span>
+ <span class="s1">'batch'</span><span class="p">,</span>
+ <span class="n">region_name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">region_name</span>
+ <span class="p">)</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">submit_job</span><span class="p">(</span>
+ <span class="n">jobName</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">,</span>
+ <span class="n">jobQueue</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="p">,</span>
+ <span class="n">jobDefinition</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">job_definition</span><span class="p">,</span>
+ <span class="n">containerOverrides</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">overrides</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'AWS Batch Job started: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">response</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">jobId</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">'jobId'</span><span class="p">]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">jobName</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">'jobName'</span><span class="p">]</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_task_ended</span><span class="p">()</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">_check_success_task</span><span class="p">()</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'AWS Batch Job has been successfully executed: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">response</span><span class="p">)</span>
+ <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'AWS Batch Job has failed executed'</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_wait_for_task_ended</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Try to use a waiter from the below pull request</span>
+
+<span class="sd"> * https://github.com/boto/botocore/pull/1307</span>
+
+<span class="sd"> If the waiter is not available apply a exponential backoff</span>
+
+<span class="sd"> * docs.aws.amazon.com/general/latest/gr/api-retries.html</span>
+<span class="sd"> """</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">waiter</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">get_waiter</span><span class="p">(</span><span class="s1">'job_execution_complete'</span><span class="p">)</span>
+ <span class="n">waiter</span><span class="o">.</span><span class="n">config</span><span class="o">.</span><span class="n">max_attempts</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">maxsize</span> <span class="c1"># timeout is managed by airflow</span>
+ <span class="n">waiter</span><span class="o">.</span><span class="n">wait</span><span class="p">(</span><span class="n">jobs</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">jobId</span><span class="p">])</span>
+ <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
+ <span class="c1"># If waiter not available use expo</span>
+ <span class="n">retry</span> <span class="o">=</span> <span class="kc">True</span>
+ <span class="n">retries</span> <span class="o">=</span> <span class="mi">0</span>
+
+ <span class="k">while</span> <span class="n">retries</span> <span class="o"><</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_retries</span> <span class="ow">or</span> <span class="n">retry</span><span class="p">:</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">describe_jobs</span><span class="p">(</span>
+ <span class="n">jobs</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">jobId</span><span class="p">]</span>
+ <span class="p">)</span>
+ <span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s1">'jobs'</span><span class="p">][</span><span class="o">-</span><span class="mi">1</span><span class="p">][</span><span class="s1">'status'</span><span class="p">]</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'SUCCEEDED'</span><span class="p">,</span> <span class="s1">'FAILED'</span><span class="p">]:</span>
+ <span class="n">retry</span> <span class="o">=</span> <span class="kc">False</span>
+
+ <span class="n">sleep</span><span class="p">(</span><span class="nb">pow</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="n">retries</span><span class="p">)</span> <span class="o">*</span> <span class="mi">100</span><span class="p">)</span>
+ <span class="n">retries</span> <span class="o">+=</span> <span class="mi">1</span>
+
+ <span class="k">def</span> <span class="nf">_check_success_task</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">describe_jobs</span><span class="p">(</span>
+ <span class="n">jobs</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">jobId</span><span class="p">],</span>
+ <span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'AWS Batch stopped, check status: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">response</span><span class="p">)</span>
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'jobs'</span><span class="p">))</span> <span class="o"><</span> <span class="mi">1</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'No job found for </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">response</span><span class="p">))</span>
+
+ <span class="k">for</span> <span class="n">job</span> <span class="ow">in</span> <span class="n">response</span><span class="p">[</span><span class="s1">'jobs'</span><span class="p">]:</span>
+ <span class="k">if</span> <span class="s1">'attempts'</span> <span class="ow">in</span> <span class="n">job</span><span class="p">:</span>
+ <span class="n">containers</span> <span class="o">=</span> <span class="n">job</span><span class="p">[</span><span class="s1">'attempts'</span><span class="p">]</span>
+ <span class="k">for</span> <span class="n">container</span> <span class="ow">in</span> <span class="n">containers</span><span class="p">:</span>
+ <span class="k">if</span> <span class="p">(</span><span class="n">job</span><span class="p">[</span><span class="s1">'status'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'FAILED'</span> <span class="ow">or</span>
+ <span class="n">container</span><span class="p">[</span><span class="s1">'container'</span><span class="p">][</span><span class="s1">'exitCode'</span><span class="p">]</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">):</span>
+ <span class="nb">print</span><span class="p">(</span><span class="s2">"@@@@"</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'This containers encounter an error during execution </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">job</span><span class="p">))</span>
+ <span class="k">elif</span> <span class="n">job</span><span class="p">[</span><span class="s1">'status'</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="s1">'SUCCEEDED'</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'This task is still pending </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">job</span><span class="p">[</span><span class="s1">'status'</span><span class="p">]))</span>
+
+ <span class="k">def</span> <span class="nf">get_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">AwsHook</span><span class="p">(</span>
+ <span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span>
+ <span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">terminate_job</span><span class="p">(</span>
+ <span class="n">jobId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">jobId</span><span class="p">,</span>
+ <span class="n">reason</span><span class="o">=</span><span class="s1">'Task killed by the user'</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">response</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/bigquery_check_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/bigquery_check_operator.html b/_modules/airflow/contrib/operators/bigquery_check_operator.html
index cbad19f..1be7f24 100644
--- a/_modules/airflow/contrib/operators/bigquery_check_operator.html
+++ b/_modules/airflow/contrib/operators/bigquery_check_operator.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,22 @@
<h1>Source code for airflow.contrib.operators.bigquery_check_operator</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="k">import</span> <span class="n">BigQueryHook</span>
<span class="kn">from</span> <span class="nn">airflow.operators.check_operator</span> <span class="k">import</span> <span class="n">CheckOperator</span><span class="p">,</span> <span class="n">ValueCheckOperator</span><span class="p">,</span> <span class="n">IntervalCheckOperator</span>
@@ -262,10 +269,10 @@
<span class="sd"> Checks that the values of metrics given as SQL expressions are within</span>
<span class="sd"> a certain tolerance of the ones from days_back before.</span>
-<span class="sd"> This method constructs a query like so:</span>
+<span class="sd"> This method constructs a query like so ::</span>
-<span class="sd"> SELECT {metrics_threshold_dict_key} FROM {table}</span>
-<span class="sd"> WHERE {date_filter_column}=<date></span>
+<span class="sd"> SELECT {metrics_threshold_dict_key} FROM {table}</span>
+<span class="sd"> WHERE {date_filter_column}=<date></span>
<span class="sd"> :param table: the table name</span>
<span class="sd"> :type table: str</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/bigquery_get_data.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/bigquery_get_data.html b/_modules/airflow/contrib/operators/bigquery_get_data.html
new file mode 100644
index 0000000..9d28a70
--- /dev/null
+++ b/_modules/airflow/contrib/operators/bigquery_get_data.html
@@ -0,0 +1,351 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.bigquery_get_data — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.bigquery_get_data</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.bigquery_get_data</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="k">import</span> <span class="n">BigQueryHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="BigQueryGetDataOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_get_data.BigQueryGetDataOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryGetDataOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Fetches the data from a BigQuery table (alternatively fetch data for selected columns)</span>
+<span class="sd"> and returns data in a python list. The number of elements in the returned list will</span>
+<span class="sd"> be equal to the number of rows fetched. Each element in the list will again be a list</span>
+<span class="sd"> where element would represent the columns values for that row.</span>
+
+<span class="sd"> **Example Result**: ``[['Tony', '10'], ['Mike', '20'], ['Steve', '15']]``</span>
+
+<span class="sd"> .. note::</span>
+<span class="sd"> If you pass fields to ``selected_fields`` which are in different order than the</span>
+<span class="sd"> order of columns already in</span>
+<span class="sd"> BQ table, the data will still be in the order of BQ table.</span>
+<span class="sd"> For example if the BQ table has 3 columns as</span>
+<span class="sd"> ``[A,B,C]`` and you pass 'B,A' in the ``selected_fields``</span>
+<span class="sd"> the data would still be of the form ``'A,B'``.</span>
+
+<span class="sd"> **Example**: ::</span>
+
+<span class="sd"> get_data = BigQueryGetDataOperator(</span>
+<span class="sd"> task_id='get_data_from_bq',</span>
+<span class="sd"> dataset_id='test_dataset',</span>
+<span class="sd"> table_id='Transaction_partitions',</span>
+<span class="sd"> max_results='100',</span>
+<span class="sd"> selected_fields='DATE',</span>
+<span class="sd"> bigquery_conn_id='airflow-service-account'</span>
+<span class="sd"> )</span>
+
+<span class="sd"> :param dataset_id: The dataset ID of the requested table.</span>
+<span class="sd"> :type destination_dataset_table: string</span>
+<span class="sd"> :param table_id: The table ID of the requested table.</span>
+<span class="sd"> :type table_id: string</span>
+<span class="sd"> :param max_results: The maximum number of records (rows) to be fetched</span>
+<span class="sd"> from the table.</span>
+<span class="sd"> :type max_results: string</span>
+<span class="sd"> :param selected_fields: List of fields to return (comma-separated). If</span>
+<span class="sd"> unspecified, all fields are returned.</span>
+<span class="sd"> :type selected_fields: string</span>
+<span class="sd"> :param bigquery_conn_id: reference to a specific BigQuery hook.</span>
+<span class="sd"> :type bigquery_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request must have domain-wide</span>
+<span class="sd"> delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'dataset_id'</span><span class="p">,</span> <span class="s1">'table_id'</span><span class="p">,</span> <span class="s1">'max_results'</span><span class="p">)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#e4f0e8'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">dataset_id</span><span class="p">,</span>
+ <span class="n">table_id</span><span class="p">,</span>
+ <span class="n">max_results</span><span class="o">=</span><span class="s1">'100'</span><span class="p">,</span>
+ <span class="n">selected_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">'bigquery_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryGetDataOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">dataset_id</span> <span class="o">=</span> <span class="n">dataset_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">table_id</span> <span class="o">=</span> <span class="n">table_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">max_results</span> <span class="o">=</span> <span class="n">max_results</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">selected_fields</span> <span class="o">=</span> <span class="n">selected_fields</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Fetching Data from:'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Dataset: </span><span class="si">%s</span><span class="s1"> ; Table: </span><span class="si">%s</span><span class="s1"> ; Max Results: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">dataset_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_results</span><span class="p">)</span>
+
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+
+ <span class="n">conn</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="n">cursor</span><span class="o">.</span><span class="n">get_tabledata</span><span class="p">(</span><span class="n">dataset_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_id</span><span class="p">,</span>
+ <span class="n">table_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_id</span><span class="p">,</span>
+ <span class="n">max_results</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">max_results</span><span class="p">,</span>
+ <span class="n">selected_fields</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">selected_fields</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Total Extracted rows: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">response</span><span class="p">[</span><span class="s1">'totalRows'</span><span class="p">])</span>
+ <span class="n">rows</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">'rows'</span><span class="p">]</span>
+
+ <span class="n">table_data</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="k">for</span> <span class="n">dict_row</span> <span class="ow">in</span> <span class="n">rows</span><span class="p">:</span>
+ <span class="n">single_row</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="k">for</span> <span class="n">fields</span> <span class="ow">in</span> <span class="n">dict_row</span><span class="p">[</span><span class="s1">'f'</span><span class="p">]:</span>
+ <span class="n">single_row</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">fields</span><span class="p">[</span><span class="s1">'v'</span><span class="p">])</span>
+ <span class="n">table_data</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">single_row</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">table_data</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[24/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/spark_jdbc_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/spark_jdbc_operator.html b/_modules/airflow/contrib/operators/spark_jdbc_operator.html
new file mode 100644
index 0000000..5a9ddab
--- /dev/null
+++ b/_modules/airflow/contrib/operators/spark_jdbc_operator.html
@@ -0,0 +1,449 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.spark_jdbc_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.spark_jdbc_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.spark_jdbc_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.operators.spark_submit_operator</span> <span class="k">import</span> <span class="n">SparkSubmitOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.spark_jdbc_hook</span> <span class="k">import</span> <span class="n">SparkJDBCHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="SparkJDBCOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.spark_jdbc_operator.SparkJDBCOperator">[docs]</a><span class="k">class</span> <span class="nc">SparkJDBCOperator</span><span class="p">(</span><span class="n">SparkSubmitOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> This operator extends the SparkSubmitOperator specifically for performing data</span>
+<span class="sd"> transfers to/from JDBC-based databases with Apache Spark. As with the</span>
+<span class="sd"> SparkSubmitOperator, it assumes that the "spark-submit" binary is available on the</span>
+<span class="sd"> PATH.</span>
+
+<span class="sd"> :param spark_app_name: Name of the job (default airflow-spark-jdbc)</span>
+<span class="sd"> :type spark_app_name: str</span>
+<span class="sd"> :param spark_conn_id: Connection id as configured in Airflow administration</span>
+<span class="sd"> :type spark_conn_id: str</span>
+<span class="sd"> :param spark_conf: Any additional Spark configuration properties</span>
+<span class="sd"> :type spark_conf: dict</span>
+<span class="sd"> :param spark_py_files: Additional python files used (.zip, .egg, or .py)</span>
+<span class="sd"> :type spark_py_files: str</span>
+<span class="sd"> :param spark_files: Additional files to upload to the container running the job</span>
+<span class="sd"> :type spark_files: str</span>
+<span class="sd"> :param spark_jars: Additional jars to upload and add to the driver and</span>
+<span class="sd"> executor classpath</span>
+<span class="sd"> :type spark_jars: str</span>
+<span class="sd"> :param num_executors: number of executor to run. This should be set so as to manage</span>
+<span class="sd"> the number of connections made with the JDBC database</span>
+<span class="sd"> :type num_executors: int</span>
+<span class="sd"> :param executor_cores: Number of cores per executor</span>
+<span class="sd"> :type executor_cores: int</span>
+<span class="sd"> :param executor_memory: Memory per executor (e.g. 1000M, 2G)</span>
+<span class="sd"> :type executor_memory: str</span>
+<span class="sd"> :param driver_memory: Memory allocated to the driver (e.g. 1000M, 2G)</span>
+<span class="sd"> :type driver_memory: str</span>
+<span class="sd"> :param verbose: Whether to pass the verbose flag to spark-submit for debugging</span>
+<span class="sd"> :type verbose: bool</span>
+<span class="sd"> :param keytab: Full path to the file that contains the keytab</span>
+<span class="sd"> :type keytab: str</span>
+<span class="sd"> :param principal: The name of the kerberos principal used for keytab</span>
+<span class="sd"> :type principal: str</span>
+<span class="sd"> :param cmd_type: Which way the data should flow. 2 possible values:</span>
+<span class="sd"> spark_to_jdbc: data written by spark from metastore to jdbc</span>
+<span class="sd"> jdbc_to_spark: data written by spark from jdbc to metastore</span>
+<span class="sd"> :type cmd_type: str</span>
+<span class="sd"> :param jdbc_table: The name of the JDBC table</span>
+<span class="sd"> :type jdbc_table: str</span>
+<span class="sd"> :param jdbc_conn_id: Connection id used for connection to JDBC database</span>
+<span class="sd"> :type: jdbc_conn_id: str</span>
+<span class="sd"> :param jdbc_driver: Name of the JDBC driver to use for the JDBC connection. This</span>
+<span class="sd"> driver (usually a jar) should be passed in the 'jars' parameter</span>
+<span class="sd"> :type jdbc_driver: str</span>
+<span class="sd"> :param metastore_table: The name of the metastore table,</span>
+<span class="sd"> :type metastore_table: str</span>
+<span class="sd"> :param jdbc_truncate: (spark_to_jdbc only) Whether or not Spark should truncate or</span>
+<span class="sd"> drop and recreate the JDBC table. This only takes effect if</span>
+<span class="sd"> 'save_mode' is set to Overwrite. Also, if the schema is</span>
+<span class="sd"> different, Spark cannot truncate, and will drop and recreate</span>
+<span class="sd"> :type jdbc_truncate: bool</span>
+<span class="sd"> :param save_mode: The Spark save-mode to use (e.g. overwrite, append, etc.)</span>
+<span class="sd"> :type save_mode: str</span>
+<span class="sd"> :param save_format: (jdbc_to_spark-only) The Spark save-format to use (e.g. parquet)</span>
+<span class="sd"> :type save_format: str</span>
+<span class="sd"> :param batch_size: (spark_to_jdbc only) The size of the batch to insert per round</span>
+<span class="sd"> trip to the JDBC database. Defaults to 1000</span>
+<span class="sd"> :type batch_size: int</span>
+<span class="sd"> :param fetch_size: (jdbc_to_spark only) The size of the batch to fetch per round trip</span>
+<span class="sd"> from the JDBC database. Default depends on the JDBC driver</span>
+<span class="sd"> :type fetch_size: int</span>
+<span class="sd"> :param num_partitions: The maximum number of partitions that can be used by Spark</span>
+<span class="sd"> simultaneously, both for spark_to_jdbc and jdbc_to_spark</span>
+<span class="sd"> operations. This will also cap the number of JDBC connections</span>
+<span class="sd"> that can be opened</span>
+<span class="sd"> :type num_partitions: int</span>
+<span class="sd"> :param partition_column: (jdbc_to_spark-only) A numeric column to be used to</span>
+<span class="sd"> partition the metastore table by. If specified, you must</span>
+<span class="sd"> also specify:</span>
+<span class="sd"> num_partitions, lower_bound, upper_bound</span>
+<span class="sd"> :type partition_column: str</span>
+<span class="sd"> :param lower_bound: (jdbc_to_spark-only) Lower bound of the range of the numeric</span>
+<span class="sd"> partition column to fetch. If specified, you must also specify:</span>
+<span class="sd"> num_partitions, partition_column, upper_bound</span>
+<span class="sd"> :type lower_bound: int</span>
+<span class="sd"> :param upper_bound: (jdbc_to_spark-only) Upper bound of the range of the numeric</span>
+<span class="sd"> partition column to fetch. If specified, you must also specify:</span>
+<span class="sd"> num_partitions, partition_column, lower_bound</span>
+<span class="sd"> :type upper_bound: int</span>
+<span class="sd"> :param create_table_column_types: (spark_to_jdbc-only) The database column data types</span>
+<span class="sd"> to use instead of the defaults, when creating the</span>
+<span class="sd"> table. Data type information should be specified in</span>
+<span class="sd"> the same format as CREATE TABLE columns syntax</span>
+<span class="sd"> (e.g: "name CHAR(64), comments VARCHAR(1024)").</span>
+<span class="sd"> The specified types should be valid spark sql data</span>
+<span class="sd"> types.</span>
+<span class="sd"> """</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">spark_app_name</span><span class="o">=</span><span class="s1">'airflow-spark-jdbc'</span><span class="p">,</span>
+ <span class="n">spark_conn_id</span><span class="o">=</span><span class="s1">'spark-default'</span><span class="p">,</span>
+ <span class="n">spark_conf</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">spark_py_files</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">spark_files</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">spark_jars</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">num_executors</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">executor_cores</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">executor_memory</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">driver_memory</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">keytab</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">principal</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">cmd_type</span><span class="o">=</span><span class="s1">'spark_to_jdbc'</span><span class="p">,</span>
+ <span class="n">jdbc_table</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">jdbc_conn_id</span><span class="o">=</span><span class="s1">'jdbc-default'</span><span class="p">,</span>
+ <span class="n">jdbc_driver</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">metastore_table</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">jdbc_truncate</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">save_mode</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">save_format</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">batch_size</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">fetch_size</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">num_partitions</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">partition_column</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">lower_bound</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">upper_bound</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">create_table_column_types</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">SparkJDBCOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_spark_app_name</span> <span class="o">=</span> <span class="n">spark_app_name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_spark_conn_id</span> <span class="o">=</span> <span class="n">spark_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_spark_conf</span> <span class="o">=</span> <span class="n">spark_conf</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_spark_py_files</span> <span class="o">=</span> <span class="n">spark_py_files</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_spark_files</span> <span class="o">=</span> <span class="n">spark_files</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_spark_jars</span> <span class="o">=</span> <span class="n">spark_jars</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_num_executors</span> <span class="o">=</span> <span class="n">num_executors</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_executor_cores</span> <span class="o">=</span> <span class="n">executor_cores</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_executor_memory</span> <span class="o">=</span> <span class="n">executor_memory</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_driver_memory</span> <span class="o">=</span> <span class="n">driver_memory</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_verbose</span> <span class="o">=</span> <span class="n">verbose</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_keytab</span> <span class="o">=</span> <span class="n">keytab</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_principal</span> <span class="o">=</span> <span class="n">principal</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_cmd_type</span> <span class="o">=</span> <span class="n">cmd_type</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_table</span> <span class="o">=</span> <span class="n">jdbc_table</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_conn_id</span> <span class="o">=</span> <span class="n">jdbc_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_driver</span> <span class="o">=</span> <span class="n">jdbc_driver</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_metastore_table</span> <span class="o">=</span> <span class="n">metastore_table</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_truncate</span> <span class="o">=</span> <span class="n">jdbc_truncate</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_save_mode</span> <span class="o">=</span> <span class="n">save_mode</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_save_format</span> <span class="o">=</span> <span class="n">save_format</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_batch_size</span> <span class="o">=</span> <span class="n">batch_size</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_fetch_size</span> <span class="o">=</span> <span class="n">fetch_size</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_num_partitions</span> <span class="o">=</span> <span class="n">num_partitions</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_partition_column</span> <span class="o">=</span> <span class="n">partition_column</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_lower_bound</span> <span class="o">=</span> <span class="n">lower_bound</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_upper_bound</span> <span class="o">=</span> <span class="n">upper_bound</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_create_table_column_types</span> <span class="o">=</span> <span class="n">create_table_column_types</span>
+
+<div class="viewcode-block" id="SparkJDBCOperator.execute"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.spark_jdbc_operator.SparkJDBCOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Call the SparkSubmitHook to run the provided spark job</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span> <span class="o">=</span> <span class="n">SparkJDBCHook</span><span class="p">(</span>
+ <span class="n">spark_app_name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_spark_app_name</span><span class="p">,</span>
+ <span class="n">spark_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_spark_conn_id</span><span class="p">,</span>
+ <span class="n">spark_conf</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_spark_conf</span><span class="p">,</span>
+ <span class="n">spark_py_files</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_spark_py_files</span><span class="p">,</span>
+ <span class="n">spark_files</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_spark_files</span><span class="p">,</span>
+ <span class="n">spark_jars</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_spark_jars</span><span class="p">,</span>
+ <span class="n">num_executors</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_num_executors</span><span class="p">,</span>
+ <span class="n">executor_cores</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_executor_cores</span><span class="p">,</span>
+ <span class="n">executor_memory</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_executor_memory</span><span class="p">,</span>
+ <span class="n">driver_memory</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_driver_memory</span><span class="p">,</span>
+ <span class="n">verbose</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_verbose</span><span class="p">,</span>
+ <span class="n">keytab</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_keytab</span><span class="p">,</span>
+ <span class="n">principal</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_principal</span><span class="p">,</span>
+ <span class="n">cmd_type</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_cmd_type</span><span class="p">,</span>
+ <span class="n">jdbc_table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_table</span><span class="p">,</span>
+ <span class="n">jdbc_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_conn_id</span><span class="p">,</span>
+ <span class="n">jdbc_driver</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_driver</span><span class="p">,</span>
+ <span class="n">metastore_table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_metastore_table</span><span class="p">,</span>
+ <span class="n">jdbc_truncate</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_truncate</span><span class="p">,</span>
+ <span class="n">save_mode</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_save_mode</span><span class="p">,</span>
+ <span class="n">save_format</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_save_format</span><span class="p">,</span>
+ <span class="n">batch_size</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_batch_size</span><span class="p">,</span>
+ <span class="n">fetch_size</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_fetch_size</span><span class="p">,</span>
+ <span class="n">num_partitions</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_num_partitions</span><span class="p">,</span>
+ <span class="n">partition_column</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_partition_column</span><span class="p">,</span>
+ <span class="n">lower_bound</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_lower_bound</span><span class="p">,</span>
+ <span class="n">upper_bound</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_upper_bound</span><span class="p">,</span>
+ <span class="n">create_table_column_types</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_create_table_column_types</span>
+ <span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span><span class="o">.</span><span class="n">submit_jdbc_job</span><span class="p">()</span></div>
+
+ <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span><span class="o">.</span><span class="n">on_kill</span><span class="p">()</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/spark_sql_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/spark_sql_operator.html b/_modules/airflow/contrib/operators/spark_sql_operator.html
new file mode 100644
index 0000000..b7583c5
--- /dev/null
+++ b/_modules/airflow/contrib/operators/spark_sql_operator.html
@@ -0,0 +1,344 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.spark_sql_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.spark_sql_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.spark_sql_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.spark_sql_hook</span> <span class="k">import</span> <span class="n">SparkSqlHook</span>
+
+
+<div class="viewcode-block" id="SparkSqlOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.spark_sql_operator.SparkSqlOperator">[docs]</a><span class="k">class</span> <span class="nc">SparkSqlOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Execute Spark SQL query</span>
+
+<span class="sd"> :param sql: The SQL query to execute</span>
+<span class="sd"> :type sql: str</span>
+<span class="sd"> :param conf: arbitrary Spark configuration property</span>
+<span class="sd"> :type conf: str (format: PROP=VALUE)</span>
+<span class="sd"> :param conn_id: connection_id string</span>
+<span class="sd"> :type conn_id: str</span>
+<span class="sd"> :param total_executor_cores: (Standalone & Mesos only) Total cores for all executors (Default: all the available cores on the worker)</span>
+<span class="sd"> :type total_executor_cores: int</span>
+<span class="sd"> :param executor_cores: (Standalone & YARN only) Number of cores per executor (Default: 2)</span>
+<span class="sd"> :type executor_cores: int</span>
+<span class="sd"> :param executor_memory: Memory per executor (e.g. 1000M, 2G) (Default: 1G)</span>
+<span class="sd"> :type executor_memory: str</span>
+<span class="sd"> :param keytab: Full path to the file that contains the keytab</span>
+<span class="sd"> :type keytab: str</span>
+<span class="sd"> :param master: spark://host:port, mesos://host:port, yarn, or local</span>
+<span class="sd"> :type master: str</span>
+<span class="sd"> :param name: Name of the job</span>
+<span class="sd"> :type name: str</span>
+<span class="sd"> :param num_executors: Number of executors to launch</span>
+<span class="sd"> :type num_executors: int</span>
+<span class="sd"> :param verbose: Whether to pass the verbose flag to spark-sql</span>
+<span class="sd"> :type verbose: bool</span>
+<span class="sd"> :param yarn_queue: The YARN queue to submit to (Default: "default")</span>
+<span class="sd"> :type yarn_queue: str</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"_sql"</span><span class="p">]</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">[</span><span class="s2">".sql"</span><span class="p">,</span> <span class="s2">".hql"</span><span class="p">]</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">sql</span><span class="p">,</span>
+ <span class="n">conf</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">conn_id</span><span class="o">=</span><span class="s1">'spark_sql_default'</span><span class="p">,</span>
+ <span class="n">total_executor_cores</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">executor_cores</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">executor_memory</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">keytab</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">principal</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">master</span><span class="o">=</span><span class="s1">'yarn'</span><span class="p">,</span>
+ <span class="n">name</span><span class="o">=</span><span class="s1">'default-name'</span><span class="p">,</span>
+ <span class="n">num_executors</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">yarn_queue</span><span class="o">=</span><span class="s1">'default'</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">SparkSqlOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_sql</span> <span class="o">=</span> <span class="n">sql</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_conf</span> <span class="o">=</span> <span class="n">conf</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_conn_id</span> <span class="o">=</span> <span class="n">conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_total_executor_cores</span> <span class="o">=</span> <span class="n">total_executor_cores</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_executor_cores</span> <span class="o">=</span> <span class="n">executor_cores</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_executor_memory</span> <span class="o">=</span> <span class="n">executor_memory</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_keytab</span> <span class="o">=</span> <span class="n">keytab</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_principal</span> <span class="o">=</span> <span class="n">principal</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_master</span> <span class="o">=</span> <span class="n">master</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="n">name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_num_executors</span> <span class="o">=</span> <span class="n">num_executors</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_yarn_queue</span> <span class="o">=</span> <span class="n">yarn_queue</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span> <span class="o">=</span> <span class="kc">None</span>
+
+<div class="viewcode-block" id="SparkSqlOperator.execute"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.spark_sql_operator.SparkSqlOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Call the SparkSqlHook to run the provided sql query</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span> <span class="o">=</span> <span class="n">SparkSqlHook</span><span class="p">(</span><span class="n">sql</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_sql</span><span class="p">,</span>
+ <span class="n">conf</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_conf</span><span class="p">,</span>
+ <span class="n">conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_conn_id</span><span class="p">,</span>
+ <span class="n">total_executor_cores</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_total_executor_cores</span><span class="p">,</span>
+ <span class="n">executor_cores</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_executor_cores</span><span class="p">,</span>
+ <span class="n">executor_memory</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_executor_memory</span><span class="p">,</span>
+ <span class="n">keytab</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_keytab</span><span class="p">,</span>
+ <span class="n">principal</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_principal</span><span class="p">,</span>
+ <span class="n">name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_name</span><span class="p">,</span>
+ <span class="n">num_executors</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_num_executors</span><span class="p">,</span>
+ <span class="n">master</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_master</span><span class="p">,</span>
+ <span class="n">yarn_queue</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_yarn_queue</span>
+ <span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span><span class="o">.</span><span class="n">run_query</span><span class="p">()</span></div>
+
+ <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span><span class="o">.</span><span class="n">kill</span><span class="p">()</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/spark_submit_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/spark_submit_operator.html b/_modules/airflow/contrib/operators/spark_submit_operator.html
new file mode 100644
index 0000000..eee6f5d
--- /dev/null
+++ b/_modules/airflow/contrib/operators/spark_submit_operator.html
@@ -0,0 +1,405 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.spark_submit_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.spark_submit_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.spark_submit_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.spark_submit_hook</span> <span class="k">import</span> <span class="n">SparkSubmitHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.settings</span> <span class="k">import</span> <span class="n">WEB_COLORS</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="SparkSubmitOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.spark_submit_operator.SparkSubmitOperator">[docs]</a><span class="k">class</span> <span class="nc">SparkSubmitOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> This hook is a wrapper around the spark-submit binary to kick off a spark-submit job.</span>
+<span class="sd"> It requires that the "spark-submit" binary is in the PATH or the spark-home is set</span>
+<span class="sd"> in the extra on the connection.</span>
+
+<span class="sd"> :param application: The application that submitted as a job, either jar or py file.</span>
+<span class="sd"> :type application: str</span>
+<span class="sd"> :param conf: Arbitrary Spark configuration properties</span>
+<span class="sd"> :type conf: dict</span>
+<span class="sd"> :param conn_id: The connection id as configured in Airflow administration. When an</span>
+<span class="sd"> invalid connection_id is supplied, it will default to yarn.</span>
+<span class="sd"> :type conn_id: str</span>
+<span class="sd"> :param files: Upload additional files to the executor running the job, separated by a</span>
+<span class="sd"> comma. Files will be placed in the working directory of each executor.</span>
+<span class="sd"> For example, serialized objects.</span>
+<span class="sd"> :type files: str</span>
+<span class="sd"> :param py_files: Additional python files used by the job, can be .zip, .egg or .py.</span>
+<span class="sd"> :type py_files: str</span>
+<span class="sd"> :param jars: Submit additional jars to upload and place them in executor classpath.</span>
+<span class="sd"> :param driver_classpath: Additional, driver-specific, classpath settings.</span>
+<span class="sd"> :type driver_classpath: str</span>
+<span class="sd"> :type jars: str</span>
+<span class="sd"> :param java_class: the main class of the Java application</span>
+<span class="sd"> :type java_class: str</span>
+<span class="sd"> :param packages: Comma-separated list of maven coordinates of jars to include on the</span>
+<span class="sd"> driver and executor classpaths</span>
+<span class="sd"> :type packages: str</span>
+<span class="sd"> :param exclude_packages: Comma-separated list of maven coordinates of jars to exclude</span>
+<span class="sd"> while resolving the dependencies provided in 'packages'</span>
+<span class="sd"> :type exclude_packages: str</span>
+<span class="sd"> :param repositories: Comma-separated list of additional remote repositories to search</span>
+<span class="sd"> for the maven coordinates given with 'packages'</span>
+<span class="sd"> :type repositories: str</span>
+<span class="sd"> :param total_executor_cores: (Standalone & Mesos only) Total cores for all executors</span>
+<span class="sd"> (Default: all the available cores on the worker)</span>
+<span class="sd"> :type total_executor_cores: int</span>
+<span class="sd"> :param executor_cores: (Standalone & YARN only) Number of cores per executor</span>
+<span class="sd"> (Default: 2)</span>
+<span class="sd"> :type executor_cores: int</span>
+<span class="sd"> :param executor_memory: Memory per executor (e.g. 1000M, 2G) (Default: 1G)</span>
+<span class="sd"> :type executor_memory: str</span>
+<span class="sd"> :param driver_memory: Memory allocated to the driver (e.g. 1000M, 2G) (Default: 1G)</span>
+<span class="sd"> :type driver_memory: str</span>
+<span class="sd"> :param keytab: Full path to the file that contains the keytab</span>
+<span class="sd"> :type keytab: str</span>
+<span class="sd"> :param principal: The name of the kerberos principal used for keytab</span>
+<span class="sd"> :type principal: str</span>
+<span class="sd"> :param name: Name of the job (default airflow-spark)</span>
+<span class="sd"> :type name: str</span>
+<span class="sd"> :param num_executors: Number of executors to launch</span>
+<span class="sd"> :type num_executors: int</span>
+<span class="sd"> :param application_args: Arguments for the application being submitted</span>
+<span class="sd"> :type application_args: list</span>
+<span class="sd"> :param env_vars: Environment variables for spark-submit. It</span>
+<span class="sd"> supports yarn and k8s mode too.</span>
+<span class="sd"> :type env_vars: dict</span>
+<span class="sd"> :param verbose: Whether to pass the verbose flag to spark-submit process for debugging</span>
+<span class="sd"> :type verbose: bool</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'_name'</span><span class="p">,</span> <span class="s1">'_application_args'</span><span class="p">,</span> <span class="s1">'_packages'</span><span class="p">)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="n">WEB_COLORS</span><span class="p">[</span><span class="s1">'LIGHTORANGE'</span><span class="p">]</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">application</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span>
+ <span class="n">conf</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">conn_id</span><span class="o">=</span><span class="s1">'spark_default'</span><span class="p">,</span>
+ <span class="n">files</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">py_files</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">driver_classpath</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">jars</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">java_class</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">packages</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">exclude_packages</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">repositories</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">total_executor_cores</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">executor_cores</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">executor_memory</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">driver_memory</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">keytab</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">principal</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">name</span><span class="o">=</span><span class="s1">'airflow-spark'</span><span class="p">,</span>
+ <span class="n">num_executors</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">application_args</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">env_vars</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">SparkSubmitOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_application</span> <span class="o">=</span> <span class="n">application</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_conf</span> <span class="o">=</span> <span class="n">conf</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_files</span> <span class="o">=</span> <span class="n">files</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_py_files</span> <span class="o">=</span> <span class="n">py_files</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_driver_classpath</span> <span class="o">=</span> <span class="n">driver_classpath</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_jars</span> <span class="o">=</span> <span class="n">jars</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_java_class</span> <span class="o">=</span> <span class="n">java_class</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_packages</span> <span class="o">=</span> <span class="n">packages</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_exclude_packages</span> <span class="o">=</span> <span class="n">exclude_packages</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_repositories</span> <span class="o">=</span> <span class="n">repositories</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_total_executor_cores</span> <span class="o">=</span> <span class="n">total_executor_cores</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_executor_cores</span> <span class="o">=</span> <span class="n">executor_cores</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_executor_memory</span> <span class="o">=</span> <span class="n">executor_memory</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_driver_memory</span> <span class="o">=</span> <span class="n">driver_memory</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_keytab</span> <span class="o">=</span> <span class="n">keytab</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_principal</span> <span class="o">=</span> <span class="n">principal</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="n">name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_num_executors</span> <span class="o">=</span> <span class="n">num_executors</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_application_args</span> <span class="o">=</span> <span class="n">application_args</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_env_vars</span> <span class="o">=</span> <span class="n">env_vars</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_verbose</span> <span class="o">=</span> <span class="n">verbose</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_conn_id</span> <span class="o">=</span> <span class="n">conn_id</span>
+
+<div class="viewcode-block" id="SparkSubmitOperator.execute"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.spark_submit_operator.SparkSubmitOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Call the SparkSubmitHook to run the provided spark job</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span> <span class="o">=</span> <span class="n">SparkSubmitHook</span><span class="p">(</span>
+ <span class="n">conf</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_conf</span><span class="p">,</span>
+ <span class="n">conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_conn_id</span><span class="p">,</span>
+ <span class="n">files</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_files</span><span class="p">,</span>
+ <span class="n">py_files</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_py_files</span><span class="p">,</span>
+ <span class="n">driver_classpath</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_driver_classpath</span><span class="p">,</span>
+ <span class="n">jars</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_jars</span><span class="p">,</span>
+ <span class="n">java_class</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_java_class</span><span class="p">,</span>
+ <span class="n">packages</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_packages</span><span class="p">,</span>
+ <span class="n">exclude_packages</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_exclude_packages</span><span class="p">,</span>
+ <span class="n">repositories</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_repositories</span><span class="p">,</span>
+ <span class="n">total_executor_cores</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_total_executor_cores</span><span class="p">,</span>
+ <span class="n">executor_cores</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_executor_cores</span><span class="p">,</span>
+ <span class="n">executor_memory</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_executor_memory</span><span class="p">,</span>
+ <span class="n">driver_memory</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_driver_memory</span><span class="p">,</span>
+ <span class="n">keytab</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_keytab</span><span class="p">,</span>
+ <span class="n">principal</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_principal</span><span class="p">,</span>
+ <span class="n">name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_name</span><span class="p">,</span>
+ <span class="n">num_executors</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_num_executors</span><span class="p">,</span>
+ <span class="n">application_args</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_application_args</span><span class="p">,</span>
+ <span class="n">env_vars</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_env_vars</span><span class="p">,</span>
+ <span class="n">verbose</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_verbose</span>
+ <span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_application</span><span class="p">)</span></div>
+
+ <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span><span class="o">.</span><span class="n">on_kill</span><span class="p">()</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[27/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/mysql_to_gcs.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/mysql_to_gcs.html b/_modules/airflow/contrib/operators/mysql_to_gcs.html
new file mode 100644
index 0000000..47346f5
--- /dev/null
+++ b/_modules/airflow/contrib/operators/mysql_to_gcs.html
@@ -0,0 +1,524 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.mysql_to_gcs — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.mysql_to_gcs</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.mysql_to_gcs</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">sys</span>
+<span class="kn">import</span> <span class="nn">json</span>
+<span class="kn">import</span> <span class="nn">time</span>
+<span class="kn">import</span> <span class="nn">base64</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcs_hook</span> <span class="k">import</span> <span class="n">GoogleCloudStorageHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="k">import</span> <span class="n">MySqlHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">date</span><span class="p">,</span> <span class="n">datetime</span>
+<span class="kn">from</span> <span class="nn">decimal</span> <span class="k">import</span> <span class="n">Decimal</span>
+<span class="kn">from</span> <span class="nn">MySQLdb.constants</span> <span class="k">import</span> <span class="n">FIELD_TYPE</span>
+<span class="kn">from</span> <span class="nn">tempfile</span> <span class="k">import</span> <span class="n">NamedTemporaryFile</span>
+<span class="kn">from</span> <span class="nn">six</span> <span class="k">import</span> <span class="n">string_types</span>
+
+<span class="n">PY3</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="mi">3</span>
+
+
+<div class="viewcode-block" id="MySqlToGoogleCloudStorageOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.mysql_to_gcs.MySqlToGoogleCloudStorageOperator">[docs]</a><span class="k">class</span> <span class="nc">MySqlToGoogleCloudStorageOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Copy data from MySQL to Google cloud storage in JSON format.</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'sql'</span><span class="p">,</span> <span class="s1">'bucket'</span><span class="p">,</span> <span class="s1">'filename'</span><span class="p">,</span> <span class="s1">'schema_filename'</span><span class="p">,</span> <span class="s1">'schema'</span><span class="p">)</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#a0e08c'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">sql</span><span class="p">,</span>
+ <span class="n">bucket</span><span class="p">,</span>
+ <span class="n">filename</span><span class="p">,</span>
+ <span class="n">schema_filename</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">approx_max_file_size_bytes</span><span class="o">=</span><span class="mi">1900000000</span><span class="p">,</span>
+ <span class="n">mysql_conn_id</span><span class="o">=</span><span class="s1">'mysql_default'</span><span class="p">,</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">schema</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> :param sql: The SQL to execute on the MySQL table.</span>
+<span class="sd"> :type sql: string</span>
+<span class="sd"> :param bucket: The bucket to upload to.</span>
+<span class="sd"> :type bucket: string</span>
+<span class="sd"> :param filename: The filename to use as the object name when uploading</span>
+<span class="sd"> to Google cloud storage. A {} should be specified in the filename</span>
+<span class="sd"> to allow the operator to inject file numbers in cases where the</span>
+<span class="sd"> file is split due to size.</span>
+<span class="sd"> :type filename: string</span>
+<span class="sd"> :param schema_filename: If set, the filename to use as the object name</span>
+<span class="sd"> when uploading a .json file containing the BigQuery schema fields</span>
+<span class="sd"> for the table that was dumped from MySQL.</span>
+<span class="sd"> :type schema_filename: string</span>
+<span class="sd"> :param approx_max_file_size_bytes: This operator supports the ability</span>
+<span class="sd"> to split large table dumps into multiple files (see notes in the</span>
+<span class="sd"> filenamed param docs above). Google cloud storage allows for files</span>
+<span class="sd"> to be a maximum of 4GB. This param allows developers to specify the</span>
+<span class="sd"> file size of the splits.</span>
+<span class="sd"> :type approx_max_file_size_bytes: long</span>
+<span class="sd"> :param mysql_conn_id: Reference to a specific MySQL hook.</span>
+<span class="sd"> :type mysql_conn_id: string</span>
+<span class="sd"> :param google_cloud_storage_conn_id: Reference to a specific Google</span>
+<span class="sd"> cloud storage hook.</span>
+<span class="sd"> :type google_cloud_storage_conn_id: string</span>
+<span class="sd"> :param schema: The schema to use, if any. Should be a list of dict or</span>
+<span class="sd"> a str. Pass a string if using Jinja template, otherwise, pass a list of</span>
+<span class="sd"> dict. Examples could be seen: https://cloud.google.com/bigquery/docs</span>
+<span class="sd"> /schemas#specifying_a_json_schema_file</span>
+<span class="sd"> :type schema: str or list</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any. For this to</span>
+<span class="sd"> work, the service account making the request must have domain-wide</span>
+<span class="sd"> delegation enabled.</span>
+<span class="sd"> """</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">MySqlToGoogleCloudStorageOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">filename</span> <span class="o">=</span> <span class="n">filename</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">schema_filename</span> <span class="o">=</span> <span class="n">schema_filename</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">approx_max_file_size_bytes</span> <span class="o">=</span> <span class="n">approx_max_file_size_bytes</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mysql_conn_id</span> <span class="o">=</span> <span class="n">mysql_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span> <span class="o">=</span> <span class="n">google_cloud_storage_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="o">=</span> <span class="n">schema</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">cursor</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_query_mysql</span><span class="p">()</span>
+ <span class="n">files_to_upload</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_write_local_data_files</span><span class="p">(</span><span class="n">cursor</span><span class="p">)</span>
+
+ <span class="c1"># If a schema is set, create a BQ schema JSON file.</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema_filename</span><span class="p">:</span>
+ <span class="n">files_to_upload</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_write_local_schema_file</span><span class="p">(</span><span class="n">cursor</span><span class="p">))</span>
+
+ <span class="c1"># Flush all files before uploading.</span>
+ <span class="k">for</span> <span class="n">file_handle</span> <span class="ow">in</span> <span class="n">files_to_upload</span><span class="o">.</span><span class="n">values</span><span class="p">():</span>
+ <span class="n">file_handle</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">_upload_to_gcs</span><span class="p">(</span><span class="n">files_to_upload</span><span class="p">)</span>
+
+ <span class="c1"># Close all temp file handles.</span>
+ <span class="k">for</span> <span class="n">file_handle</span> <span class="ow">in</span> <span class="n">files_to_upload</span><span class="o">.</span><span class="n">values</span><span class="p">():</span>
+ <span class="n">file_handle</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
+
+ <span class="k">def</span> <span class="nf">_query_mysql</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Queries mysql and returns a cursor to the results.</span>
+<span class="sd"> """</span>
+ <span class="n">mysql</span> <span class="o">=</span> <span class="n">MySqlHook</span><span class="p">(</span><span class="n">mysql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mysql_conn_id</span><span class="p">)</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="n">mysql</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
+ <span class="n">cursor</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">cursor</span>
+
+ <span class="k">def</span> <span class="nf">_write_local_data_files</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cursor</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Takes a cursor, and writes results to a local file.</span>
+
+<span class="sd"> :return: A dictionary where keys are filenames to be used as object</span>
+<span class="sd"> names in GCS, and values are file handles to local files that</span>
+<span class="sd"> contain the data for the GCS objects.</span>
+<span class="sd"> """</span>
+ <span class="n">schema</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">schema_tuple</span><span class="p">:</span> <span class="n">schema_tuple</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">cursor</span><span class="o">.</span><span class="n">description</span><span class="p">))</span>
+ <span class="n">col_type_dict</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_col_type_dict</span><span class="p">()</span>
+ <span class="n">file_no</span> <span class="o">=</span> <span class="mi">0</span>
+ <span class="n">tmp_file_handle</span> <span class="o">=</span> <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="n">delete</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="n">tmp_file_handles</span> <span class="o">=</span> <span class="p">{</span><span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">file_no</span><span class="p">):</span> <span class="n">tmp_file_handle</span><span class="p">}</span>
+
+ <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">cursor</span><span class="p">:</span>
+ <span class="c1"># Convert datetime objects to utc seconds, and decimals to floats.</span>
+ <span class="c1"># Convert binary type object to string encoded with base64.</span>
+ <span class="n">row</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_convert_types</span><span class="p">(</span><span class="n">schema</span><span class="p">,</span> <span class="n">col_type_dict</span><span class="p">,</span> <span class="n">row</span><span class="p">)</span>
+ <span class="n">row_dict</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">schema</span><span class="p">,</span> <span class="n">row</span><span class="p">))</span>
+
+ <span class="c1"># TODO validate that row isn't > 2MB. BQ enforces a hard row size of 2MB.</span>
+ <span class="n">s</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">row_dict</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">PY3</span><span class="p">:</span>
+ <span class="n">s</span> <span class="o">=</span> <span class="n">s</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span>
+ <span class="n">tmp_file_handle</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">s</span><span class="p">)</span>
+
+ <span class="c1"># Append newline to make dumps BigQuery compatible.</span>
+ <span class="n">tmp_file_handle</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="sa">b</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">)</span>
+
+ <span class="c1"># Stop if the file exceeds the file size limit.</span>
+ <span class="k">if</span> <span class="n">tmp_file_handle</span><span class="o">.</span><span class="n">tell</span><span class="p">()</span> <span class="o">>=</span> <span class="bp">self</span><span class="o">.</span><span class="n">approx_max_file_size_bytes</span><span class="p">:</span>
+ <span class="n">file_no</span> <span class="o">+=</span> <span class="mi">1</span>
+ <span class="n">tmp_file_handle</span> <span class="o">=</span> <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="n">delete</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="n">tmp_file_handles</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">file_no</span><span class="p">)]</span> <span class="o">=</span> <span class="n">tmp_file_handle</span>
+
+ <span class="k">return</span> <span class="n">tmp_file_handles</span>
+
+ <span class="k">def</span> <span class="nf">_write_local_schema_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cursor</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Takes a cursor, and writes the BigQuery schema for the results to a</span>
+<span class="sd"> local file system.</span>
+
+<span class="sd"> :return: A dictionary where key is a filename to be used as an object</span>
+<span class="sd"> name in GCS, and values are file handles to local files that</span>
+<span class="sd"> contains the BigQuery schema fields in .json format.</span>
+<span class="sd"> """</span>
+ <span class="n">schema_str</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="n">tmp_schema_file_handle</span> <span class="o">=</span> <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="n">delete</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="n">string_types</span><span class="p">):</span>
+ <span class="n">schema_str</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span>
+ <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+ <span class="n">schema_str</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">schema</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="k">for</span> <span class="n">field</span> <span class="ow">in</span> <span class="n">cursor</span><span class="o">.</span><span class="n">description</span><span class="p">:</span>
+ <span class="c1"># See PEP 249 for details about the description tuple.</span>
+ <span class="n">field_name</span> <span class="o">=</span> <span class="n">field</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+ <span class="n">field_type</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">type_map</span><span class="p">(</span><span class="n">field</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
+ <span class="c1"># Always allow TIMESTAMP to be nullable. MySQLdb returns None types</span>
+ <span class="c1"># for required fields because some MySQL timestamps can't be</span>
+ <span class="c1"># represented by Python's datetime (e.g. 0000-00-00 00:00:00).</span>
+ <span class="k">if</span> <span class="n">field</span><span class="p">[</span><span class="mi">6</span><span class="p">]</span> <span class="ow">or</span> <span class="n">field_type</span> <span class="o">==</span> <span class="s1">'TIMESTAMP'</span><span class="p">:</span>
+ <span class="n">field_mode</span> <span class="o">=</span> <span class="s1">'NULLABLE'</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">field_mode</span> <span class="o">=</span> <span class="s1">'REQUIRED'</span>
+ <span class="n">schema</span><span class="o">.</span><span class="n">append</span><span class="p">({</span>
+ <span class="s1">'name'</span><span class="p">:</span> <span class="n">field_name</span><span class="p">,</span>
+ <span class="s1">'type'</span><span class="p">:</span> <span class="n">field_type</span><span class="p">,</span>
+ <span class="s1">'mode'</span><span class="p">:</span> <span class="n">field_mode</span><span class="p">,</span>
+ <span class="p">})</span>
+ <span class="n">schema_str</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">schema</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">PY3</span><span class="p">:</span>
+ <span class="n">schema_str</span> <span class="o">=</span> <span class="n">schema_str</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span>
+ <span class="n">tmp_schema_file_handle</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">schema_str</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Using schema for </span><span class="si">%s</span><span class="s1">: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema_filename</span><span class="p">,</span> <span class="n">schema_str</span><span class="p">)</span>
+ <span class="k">return</span> <span class="p">{</span><span class="bp">self</span><span class="o">.</span><span class="n">schema_filename</span><span class="p">:</span> <span class="n">tmp_schema_file_handle</span><span class="p">}</span>
+
+ <span class="k">def</span> <span class="nf">_upload_to_gcs</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">files_to_upload</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Upload all of the file splits (and optionally the schema .json file) to</span>
+<span class="sd"> Google cloud storage.</span>
+<span class="sd"> """</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+ <span class="k">for</span> <span class="nb">object</span><span class="p">,</span> <span class="n">tmp_file_handle</span> <span class="ow">in</span> <span class="n">files_to_upload</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+ <span class="n">hook</span><span class="o">.</span><span class="n">upload</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">tmp_file_handle</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="s1">'application/json'</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_convert_types</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span> <span class="n">col_type_dict</span><span class="p">,</span> <span class="n">row</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Takes a value from MySQLdb, and converts it to a value that's safe for</span>
+<span class="sd"> JSON/Google cloud storage/BigQuery. Dates are converted to UTC seconds.</span>
+<span class="sd"> Decimals are converted to floats. Binary type fields are encoded with base64,</span>
+<span class="sd"> as imported BYTES data must be base64-encoded according to Bigquery SQL</span>
+<span class="sd"> date type documentation: https://cloud.google.com/bigquery/data-types</span>
+<span class="sd"> """</span>
+ <span class="n">converted_row</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="k">for</span> <span class="n">col_name</span><span class="p">,</span> <span class="n">col_val</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">schema</span><span class="p">,</span> <span class="n">row</span><span class="p">):</span>
+ <span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">col_val</span><span class="p">)</span> <span class="ow">in</span> <span class="p">(</span><span class="n">datetime</span><span class="p">,</span> <span class="n">date</span><span class="p">):</span>
+ <span class="n">col_val</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">mktime</span><span class="p">(</span><span class="n">col_val</span><span class="o">.</span><span class="n">timetuple</span><span class="p">())</span>
+ <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col_val</span><span class="p">,</span> <span class="n">Decimal</span><span class="p">):</span>
+ <span class="n">col_val</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">col_val</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="n">col_type_dict</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">col_name</span><span class="p">)</span> <span class="o">==</span> <span class="s2">"BYTES"</span><span class="p">:</span>
+ <span class="n">col_val</span> <span class="o">=</span> <span class="n">base64</span><span class="o">.</span><span class="n">urlsafe_b64encode</span><span class="p">(</span><span class="n">col_val</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">PY3</span><span class="p">:</span>
+ <span class="n">col_val</span> <span class="o">=</span> <span class="n">col_val</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'ascii'</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">col_val</span> <span class="o">=</span> <span class="n">col_val</span>
+ <span class="n">converted_row</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">col_val</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">converted_row</span>
+
+ <span class="k">def</span> <span class="nf">_get_col_type_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Return a dict of column name and column type based on self.schema if not None.</span>
+<span class="sd"> """</span>
+ <span class="n">schema</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="n">string_types</span><span class="p">):</span>
+ <span class="n">schema</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+ <span class="n">schema</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span>
+ <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s1">'Using default schema due to unexpected type.'</span>
+ <span class="s1">'Should be a string or list.'</span><span class="p">)</span>
+
+ <span class="n">col_type_dict</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">col_type_dict</span> <span class="o">=</span> <span class="p">{</span><span class="n">col</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]:</span> <span class="n">col</span><span class="p">[</span><span class="s1">'type'</span><span class="p">]</span> <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">schema</span><span class="p">}</span>
+ <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s1">'Using default schema due to missing name or type. Please '</span>
+ <span class="s1">'refer to: https://cloud.google.com/bigquery/docs/schemas'</span>
+ <span class="s1">'#specifying_a_json_schema_file'</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">col_type_dict</span>
+
+<div class="viewcode-block" id="MySqlToGoogleCloudStorageOperator.type_map"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.mysql_to_gcs.MySqlToGoogleCloudStorageOperator.type_map">[docs]</a> <span class="nd">@classmethod</span>
+ <span class="k">def</span> <span class="nf">type_map</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">mysql_type</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Helper function that maps from MySQL fields to BigQuery fields. Used</span>
+<span class="sd"> when a schema_filename is set.</span>
+<span class="sd"> """</span>
+ <span class="n">d</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="n">FIELD_TYPE</span><span class="o">.</span><span class="n">INT24</span><span class="p">:</span> <span class="s1">'INTEGER'</span><span class="p">,</span>
+ <span class="n">FIELD_TYPE</span><span class="o">.</span><span class="n">TINY</span><span class="p">:</span> <span class="s1">'INTEGER'</span><span class="p">,</span>
+ <span class="n">FIELD_TYPE</span><span class="o">.</span><span class="n">BIT</span><span class="p">:</span> <span class="s1">'INTEGER'</span><span class="p">,</span>
+ <span class="n">FIELD_TYPE</span><span class="o">.</span><span class="n">DATETIME</span><span class="p">:</span> <span class="s1">'TIMESTAMP'</span><span class="p">,</span>
+ <span class="n">FIELD_TYPE</span><span class="o">.</span><span class="n">DATE</span><span class="p">:</span> <span class="s1">'TIMESTAMP'</span><span class="p">,</span>
+ <span class="n">FIELD_TYPE</span><span class="o">.</span><span class="n">DECIMAL</span><span class="p">:</span> <span class="s1">'FLOAT'</span><span class="p">,</span>
+ <span class="n">FIELD_TYPE</span><span class="o">.</span><span class="n">NEWDECIMAL</span><span class="p">:</span> <span class="s1">'FLOAT'</span><span class="p">,</span>
+ <span class="n">FIELD_TYPE</span><span class="o">.</span><span class="n">DOUBLE</span><span class="p">:</span> <span class="s1">'FLOAT'</span><span class="p">,</span>
+ <span class="n">FIELD_TYPE</span><span class="o">.</span><span class="n">FLOAT</span><span class="p">:</span> <span class="s1">'FLOAT'</span><span class="p">,</span>
+ <span class="n">FIELD_TYPE</span><span class="o">.</span><span class="n">INT24</span><span class="p">:</span> <span class="s1">'INTEGER'</span><span class="p">,</span>
+ <span class="n">FIELD_TYPE</span><span class="o">.</span><span class="n">LONG</span><span class="p">:</span> <span class="s1">'INTEGER'</span><span class="p">,</span>
+ <span class="n">FIELD_TYPE</span><span class="o">.</span><span class="n">LONGLONG</span><span class="p">:</span> <span class="s1">'INTEGER'</span><span class="p">,</span>
+ <span class="n">FIELD_TYPE</span><span class="o">.</span><span class="n">SHORT</span><span class="p">:</span> <span class="s1">'INTEGER'</span><span class="p">,</span>
+ <span class="n">FIELD_TYPE</span><span class="o">.</span><span class="n">TIMESTAMP</span><span class="p">:</span> <span class="s1">'TIMESTAMP'</span><span class="p">,</span>
+ <span class="n">FIELD_TYPE</span><span class="o">.</span><span class="n">YEAR</span><span class="p">:</span> <span class="s1">'INTEGER'</span><span class="p">,</span>
+ <span class="p">}</span>
+ <span class="k">return</span> <span class="n">d</span><span class="p">[</span><span class="n">mysql_type</span><span class="p">]</span> <span class="k">if</span> <span class="n">mysql_type</span> <span class="ow">in</span> <span class="n">d</span> <span class="k">else</span> <span class="s1">'STRING'</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/postgres_to_gcs_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/postgres_to_gcs_operator.html b/_modules/airflow/contrib/operators/postgres_to_gcs_operator.html
new file mode 100644
index 0000000..10c7d8f
--- /dev/null
+++ b/_modules/airflow/contrib/operators/postgres_to_gcs_operator.html
@@ -0,0 +1,481 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.postgres_to_gcs_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.postgres_to_gcs_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.postgres_to_gcs_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">sys</span>
+<span class="kn">import</span> <span class="nn">json</span>
+<span class="kn">import</span> <span class="nn">time</span>
+<span class="kn">import</span> <span class="nn">datetime</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcs_hook</span> <span class="k">import</span> <span class="n">GoogleCloudStorageHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.postgres_hook</span> <span class="k">import</span> <span class="n">PostgresHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">decimal</span> <span class="k">import</span> <span class="n">Decimal</span>
+<span class="kn">from</span> <span class="nn">tempfile</span> <span class="k">import</span> <span class="n">NamedTemporaryFile</span>
+
+<span class="n">PY3</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="mi">3</span>
+
+
+<div class="viewcode-block" id="PostgresToGoogleCloudStorageOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.postgres_to_gcs_operator.PostgresToGoogleCloudStorageOperator">[docs]</a><span class="k">class</span> <span class="nc">PostgresToGoogleCloudStorageOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Copy data from Postgres to Google Cloud Storage in JSON format.</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'sql'</span><span class="p">,</span> <span class="s1">'bucket'</span><span class="p">,</span> <span class="s1">'filename'</span><span class="p">,</span> <span class="s1">'schema_filename'</span><span class="p">,</span>
+ <span class="s1">'parameters'</span><span class="p">)</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,</span> <span class="p">)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#a0e08c'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">sql</span><span class="p">,</span>
+ <span class="n">bucket</span><span class="p">,</span>
+ <span class="n">filename</span><span class="p">,</span>
+ <span class="n">schema_filename</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">approx_max_file_size_bytes</span><span class="o">=</span><span class="mi">1900000000</span><span class="p">,</span>
+ <span class="n">postgres_conn_id</span><span class="o">=</span><span class="s1">'postgres_default'</span><span class="p">,</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> :param sql: The SQL to execute on the Postgres table.</span>
+<span class="sd"> :type sql: string</span>
+<span class="sd"> :param bucket: The bucket to upload to.</span>
+<span class="sd"> :type bucket: string</span>
+<span class="sd"> :param filename: The filename to use as the object name when uploading</span>
+<span class="sd"> to Google Cloud Storage. A {} should be specified in the filename</span>
+<span class="sd"> to allow the operator to inject file numbers in cases where the</span>
+<span class="sd"> file is split due to size.</span>
+<span class="sd"> :type filename: string</span>
+<span class="sd"> :param schema_filename: If set, the filename to use as the object name</span>
+<span class="sd"> when uploading a .json file containing the BigQuery schema fields</span>
+<span class="sd"> for the table that was dumped from Postgres.</span>
+<span class="sd"> :type schema_filename: string</span>
+<span class="sd"> :param approx_max_file_size_bytes: This operator supports the ability</span>
+<span class="sd"> to split large table dumps into multiple files (see notes in the</span>
+<span class="sd"> filenamed param docs above). Google Cloud Storage allows for files</span>
+<span class="sd"> to be a maximum of 4GB. This param allows developers to specify the</span>
+<span class="sd"> file size of the splits.</span>
+<span class="sd"> :type approx_max_file_size_bytes: long</span>
+<span class="sd"> :param postgres_conn_id: Reference to a specific Postgres hook.</span>
+<span class="sd"> :type postgres_conn_id: string</span>
+<span class="sd"> :param google_cloud_storage_conn_id: Reference to a specific Google</span>
+<span class="sd"> cloud storage hook.</span>
+<span class="sd"> :type google_cloud_storage_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any. For this to</span>
+<span class="sd"> work, the service account making the request must have domain-wide</span>
+<span class="sd"> delegation enabled.</span>
+<span class="sd"> :param parameters: a parameters dict that is substituted at query runtime.</span>
+<span class="sd"> :type parameters: dict</span>
+<span class="sd"> """</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">PostgresToGoogleCloudStorageOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">filename</span> <span class="o">=</span> <span class="n">filename</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">schema_filename</span> <span class="o">=</span> <span class="n">schema_filename</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">approx_max_file_size_bytes</span> <span class="o">=</span> <span class="n">approx_max_file_size_bytes</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">postgres_conn_id</span> <span class="o">=</span> <span class="n">postgres_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span> <span class="o">=</span> <span class="n">google_cloud_storage_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span> <span class="o">=</span> <span class="n">parameters</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">cursor</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_query_postgres</span><span class="p">()</span>
+ <span class="n">files_to_upload</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_write_local_data_files</span><span class="p">(</span><span class="n">cursor</span><span class="p">)</span>
+
+ <span class="c1"># If a schema is set, create a BQ schema JSON file.</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema_filename</span><span class="p">:</span>
+ <span class="n">files_to_upload</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_write_local_schema_file</span><span class="p">(</span><span class="n">cursor</span><span class="p">))</span>
+
+ <span class="c1"># Flush all files before uploading</span>
+ <span class="k">for</span> <span class="n">file_handle</span> <span class="ow">in</span> <span class="n">files_to_upload</span><span class="o">.</span><span class="n">values</span><span class="p">():</span>
+ <span class="n">file_handle</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">_upload_to_gcs</span><span class="p">(</span><span class="n">files_to_upload</span><span class="p">)</span>
+
+ <span class="c1"># Close all temp file handles.</span>
+ <span class="k">for</span> <span class="n">file_handle</span> <span class="ow">in</span> <span class="n">files_to_upload</span><span class="o">.</span><span class="n">values</span><span class="p">():</span>
+ <span class="n">file_handle</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
+
+ <span class="k">def</span> <span class="nf">_query_postgres</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Queries Postgres and returns a cursor to the results.</span>
+<span class="sd"> """</span>
+ <span class="n">postgres</span> <span class="o">=</span> <span class="n">PostgresHook</span><span class="p">(</span><span class="n">postgres_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">postgres_conn_id</span><span class="p">)</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="n">postgres</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
+ <span class="n">cursor</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">cursor</span>
+
+ <span class="k">def</span> <span class="nf">_write_local_data_files</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cursor</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Takes a cursor, and writes results to a local file.</span>
+
+<span class="sd"> :return: A dictionary where keys are filenames to be used as object</span>
+<span class="sd"> names in GCS, and values are file handles to local files that</span>
+<span class="sd"> contain the data for the GCS objects.</span>
+<span class="sd"> """</span>
+ <span class="n">schema</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">schema_tuple</span><span class="p">:</span> <span class="n">schema_tuple</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">cursor</span><span class="o">.</span><span class="n">description</span><span class="p">))</span>
+ <span class="n">file_no</span> <span class="o">=</span> <span class="mi">0</span>
+ <span class="n">tmp_file_handle</span> <span class="o">=</span> <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="n">delete</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="n">tmp_file_handles</span> <span class="o">=</span> <span class="p">{</span><span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">file_no</span><span class="p">):</span> <span class="n">tmp_file_handle</span><span class="p">}</span>
+
+ <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">cursor</span><span class="p">:</span>
+ <span class="c1"># Convert datetime objects to utc seconds, and decimals to floats</span>
+ <span class="n">row</span> <span class="o">=</span> <span class="nb">map</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">convert_types</span><span class="p">,</span> <span class="n">row</span><span class="p">)</span>
+ <span class="n">row_dict</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">schema</span><span class="p">,</span> <span class="n">row</span><span class="p">))</span>
+
+ <span class="n">s</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">row_dict</span><span class="p">,</span> <span class="n">sort_keys</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">PY3</span><span class="p">:</span>
+ <span class="n">s</span> <span class="o">=</span> <span class="n">s</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span>
+ <span class="n">tmp_file_handle</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">s</span><span class="p">)</span>
+
+ <span class="c1"># Append newline to make dumps BigQuery compatible.</span>
+ <span class="n">tmp_file_handle</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="sa">b</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">)</span>
+
+ <span class="c1"># Stop if the file exceeds the file size limit.</span>
+ <span class="k">if</span> <span class="n">tmp_file_handle</span><span class="o">.</span><span class="n">tell</span><span class="p">()</span> <span class="o">>=</span> <span class="bp">self</span><span class="o">.</span><span class="n">approx_max_file_size_bytes</span><span class="p">:</span>
+ <span class="n">file_no</span> <span class="o">+=</span> <span class="mi">1</span>
+ <span class="n">tmp_file_handle</span> <span class="o">=</span> <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="n">delete</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="n">tmp_file_handles</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">file_no</span><span class="p">)]</span> <span class="o">=</span> <span class="n">tmp_file_handle</span>
+
+ <span class="k">return</span> <span class="n">tmp_file_handles</span>
+
+ <span class="k">def</span> <span class="nf">_write_local_schema_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cursor</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Takes a cursor, and writes the BigQuery schema for the results to a</span>
+<span class="sd"> local file system.</span>
+
+<span class="sd"> :return: A dictionary where key is a filename to be used as an object</span>
+<span class="sd"> name in GCS, and values are file handles to local files that</span>
+<span class="sd"> contains the BigQuery schema fields in .json format.</span>
+<span class="sd"> """</span>
+ <span class="n">schema</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="k">for</span> <span class="n">field</span> <span class="ow">in</span> <span class="n">cursor</span><span class="o">.</span><span class="n">description</span><span class="p">:</span>
+ <span class="c1"># See PEP 249 for details about the description tuple.</span>
+ <span class="n">field_name</span> <span class="o">=</span> <span class="n">field</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+ <span class="n">field_type</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">type_map</span><span class="p">(</span><span class="n">field</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
+ <span class="n">field_mode</span> <span class="o">=</span> <span class="s1">'REPEATED'</span> <span class="k">if</span> <span class="n">field</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="ow">in</span> <span class="p">(</span><span class="mi">1009</span><span class="p">,</span> <span class="mi">1005</span><span class="p">,</span> <span class="mi">1007</span><span class="p">,</span>
+ <span class="mi">1016</span><span class="p">)</span> <span class="k">else</span> <span class="s1">'NULLABLE'</span>
+ <span class="n">schema</span><span class="o">.</span><span class="n">append</span><span class="p">({</span>
+ <span class="s1">'name'</span><span class="p">:</span> <span class="n">field_name</span><span class="p">,</span>
+ <span class="s1">'type'</span><span class="p">:</span> <span class="n">field_type</span><span class="p">,</span>
+ <span class="s1">'mode'</span><span class="p">:</span> <span class="n">field_mode</span><span class="p">,</span>
+ <span class="p">})</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Using schema for </span><span class="si">%s</span><span class="s1">: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema_filename</span><span class="p">,</span> <span class="n">schema</span><span class="p">)</span>
+ <span class="n">tmp_schema_file_handle</span> <span class="o">=</span> <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="n">delete</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="n">s</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">schema</span><span class="p">,</span> <span class="n">sort_keys</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">PY3</span><span class="p">:</span>
+ <span class="n">s</span> <span class="o">=</span> <span class="n">s</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span>
+ <span class="n">tmp_schema_file_handle</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">s</span><span class="p">)</span>
+ <span class="k">return</span> <span class="p">{</span><span class="bp">self</span><span class="o">.</span><span class="n">schema_filename</span><span class="p">:</span> <span class="n">tmp_schema_file_handle</span><span class="p">}</span>
+
+ <span class="k">def</span> <span class="nf">_upload_to_gcs</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">files_to_upload</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Upload all of the file splits (and optionally the schema .json file) to</span>
+<span class="sd"> Google Cloud Storage.</span>
+<span class="sd"> """</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+ <span class="k">for</span> <span class="nb">object</span><span class="p">,</span> <span class="n">tmp_file_handle</span> <span class="ow">in</span> <span class="n">files_to_upload</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+ <span class="n">hook</span><span class="o">.</span><span class="n">upload</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">tmp_file_handle</span><span class="o">.</span><span class="n">name</span><span class="p">,</span>
+ <span class="s1">'application/json'</span><span class="p">)</span>
+
+<div class="viewcode-block" id="PostgresToGoogleCloudStorageOperator.convert_types"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.postgres_to_gcs_operator.PostgresToGoogleCloudStorageOperator.convert_types">[docs]</a> <span class="nd">@classmethod</span>
+ <span class="k">def</span> <span class="nf">convert_types</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Takes a value from Postgres, and converts it to a value that's safe for</span>
+<span class="sd"> JSON/Google Cloud Storage/BigQuery. Dates are converted to UTC seconds.</span>
+<span class="sd"> Decimals are converted to floats. Times are converted to seconds.</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">value</span><span class="p">)</span> <span class="ow">in</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="p">,</span> <span class="n">datetime</span><span class="o">.</span><span class="n">date</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">time</span><span class="o">.</span><span class="n">mktime</span><span class="p">(</span><span class="n">value</span><span class="o">.</span><span class="n">timetuple</span><span class="p">())</span>
+ <span class="k">elif</span> <span class="nb">type</span><span class="p">(</span><span class="n">value</span><span class="p">)</span> <span class="o">==</span> <span class="n">datetime</span><span class="o">.</span><span class="n">time</span><span class="p">:</span>
+ <span class="n">formated_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">value</span><span class="p">),</span> <span class="s2">"%H:%M:%S"</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">timedelta</span><span class="p">(</span>
+ <span class="n">hours</span><span class="o">=</span><span class="n">formated_time</span><span class="o">.</span><span class="n">tm_hour</span><span class="p">,</span>
+ <span class="n">minutes</span><span class="o">=</span><span class="n">formated_time</span><span class="o">.</span><span class="n">tm_min</span><span class="p">,</span>
+ <span class="n">seconds</span><span class="o">=</span><span class="n">formated_time</span><span class="o">.</span><span class="n">tm_sec</span><span class="p">)</span><span class="o">.</span><span class="n">seconds</span>
+ <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">Decimal</span><span class="p">):</span>
+ <span class="k">return</span> <span class="nb">float</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">value</span></div>
+
+<div class="viewcode-block" id="PostgresToGoogleCloudStorageOperator.type_map"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.postgres_to_gcs_operator.PostgresToGoogleCloudStorageOperator.type_map">[docs]</a> <span class="nd">@classmethod</span>
+ <span class="k">def</span> <span class="nf">type_map</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">postgres_type</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Helper function that maps from Postgres fields to BigQuery fields. Used</span>
+<span class="sd"> when a schema_filename is set.</span>
+<span class="sd"> """</span>
+ <span class="n">d</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="mi">1114</span><span class="p">:</span> <span class="s1">'TIMESTAMP'</span><span class="p">,</span>
+ <span class="mi">1184</span><span class="p">:</span> <span class="s1">'TIMESTAMP'</span><span class="p">,</span>
+ <span class="mi">1082</span><span class="p">:</span> <span class="s1">'TIMESTAMP'</span><span class="p">,</span>
+ <span class="mi">1083</span><span class="p">:</span> <span class="s1">'TIMESTAMP'</span><span class="p">,</span>
+ <span class="mi">1005</span><span class="p">:</span> <span class="s1">'INTEGER'</span><span class="p">,</span>
+ <span class="mi">1007</span><span class="p">:</span> <span class="s1">'INTEGER'</span><span class="p">,</span>
+ <span class="mi">1016</span><span class="p">:</span> <span class="s1">'INTEGER'</span><span class="p">,</span>
+ <span class="mi">20</span><span class="p">:</span> <span class="s1">'INTEGER'</span><span class="p">,</span>
+ <span class="mi">21</span><span class="p">:</span> <span class="s1">'INTEGER'</span><span class="p">,</span>
+ <span class="mi">23</span><span class="p">:</span> <span class="s1">'INTEGER'</span><span class="p">,</span>
+ <span class="mi">16</span><span class="p">:</span> <span class="s1">'BOOLEAN'</span><span class="p">,</span>
+ <span class="mi">700</span><span class="p">:</span> <span class="s1">'FLOAT'</span><span class="p">,</span>
+ <span class="mi">701</span><span class="p">:</span> <span class="s1">'FLOAT'</span><span class="p">,</span>
+ <span class="mi">1700</span><span class="p">:</span> <span class="s1">'FLOAT'</span>
+ <span class="p">}</span>
+
+ <span class="k">return</span> <span class="n">d</span><span class="p">[</span><span class="n">postgres_type</span><span class="p">]</span> <span class="k">if</span> <span class="n">postgres_type</span> <span class="ow">in</span> <span class="n">d</span> <span class="k">else</span> <span class="s1">'STRING'</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[25/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/s3_to_gcs_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/s3_to_gcs_operator.html b/_modules/airflow/contrib/operators/s3_to_gcs_operator.html
new file mode 100644
index 0000000..754fcb2
--- /dev/null
+++ b/_modules/airflow/contrib/operators/s3_to_gcs_operator.html
@@ -0,0 +1,425 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.s3_to_gcs_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.s3_to_gcs_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.s3_to_gcs_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">tempfile</span> <span class="k">import</span> <span class="n">NamedTemporaryFile</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcs_hook</span> <span class="k">import</span> <span class="p">(</span><span class="n">GoogleCloudStorageHook</span><span class="p">,</span>
+ <span class="n">_parse_gcs_url</span><span class="p">)</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.operators.s3_list_operator</span> <span class="k">import</span> <span class="n">S3ListOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.S3_hook</span> <span class="k">import</span> <span class="n">S3Hook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="S3ToGoogleCloudStorageOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.s3_to_gcs_operator.S3ToGoogleCloudStorageOperator">[docs]</a><span class="k">class</span> <span class="nc">S3ToGoogleCloudStorageOperator</span><span class="p">(</span><span class="n">S3ListOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Synchronizes an S3 key, possibly a prefix, with a Google Cloud Storage</span>
+<span class="sd"> destination path.</span>
+
+<span class="sd"> :param bucket: The S3 bucket where to find the objects.</span>
+<span class="sd"> :type bucket: string</span>
+<span class="sd"> :param prefix: Prefix string which filters objects whose name begin with</span>
+<span class="sd"> such prefix.</span>
+<span class="sd"> :type prefix: string</span>
+<span class="sd"> :param delimiter: the delimiter marks key hierarchy.</span>
+<span class="sd"> :type delimiter: string</span>
+<span class="sd"> :param aws_conn_id: The source S3 connection</span>
+<span class="sd"> :type aws_conn_id: string</span>
+<span class="sd"> :param dest_gcs_conn_id: The destination connection ID to use</span>
+<span class="sd"> when connecting to Google Cloud Storage.</span>
+<span class="sd"> :type dest_gcs_conn_id: string</span>
+<span class="sd"> :param dest_gcs: The destination Google Cloud Storage bucket and prefix</span>
+<span class="sd"> where you want to store the files.</span>
+<span class="sd"> :type dest_gcs: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request must have</span>
+<span class="sd"> domain-wide delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> :param replace: Whether you want to replace existing destination files</span>
+<span class="sd"> or not.</span>
+<span class="sd"> :type replace: bool</span>
+
+
+<span class="sd"> **Example**:</span>
+<span class="sd"> .. code-block:: python</span>
+<span class="sd"> s3_to_gcs_op = S3ToGoogleCloudStorageOperator(</span>
+<span class="sd"> task_id='s3_to_gcs_example',</span>
+<span class="sd"> bucket='my-s3-bucket',</span>
+<span class="sd"> prefix='data/customers-201804',</span>
+<span class="sd"> dest_gcs_conn_id='google_cloud_default',</span>
+<span class="sd"> dest_gcs='gs://my.gcs.bucket/some/customers/',</span>
+<span class="sd"> replace=False,</span>
+<span class="sd"> dag=my-dag)</span>
+
+<span class="sd"> Note that ``bucket``, ``prefix``, ``delimiter`` and ``dest_gcs`` are</span>
+<span class="sd"> templated, so you can use variables in them if you wish.</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bucket'</span><span class="p">,</span> <span class="s1">'prefix'</span><span class="p">,</span> <span class="s1">'delimiter'</span><span class="p">,</span> <span class="s1">'dest_gcs'</span><span class="p">)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#e09411'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">bucket</span><span class="p">,</span>
+ <span class="n">prefix</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span>
+ <span class="n">delimiter</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span>
+ <span class="n">aws_conn_id</span><span class="o">=</span><span class="s1">'aws_default'</span><span class="p">,</span>
+ <span class="n">dest_gcs_conn_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">dest_gcs</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">replace</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+
+ <span class="nb">super</span><span class="p">(</span><span class="n">S3ToGoogleCloudStorageOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+ <span class="n">bucket</span><span class="o">=</span><span class="n">bucket</span><span class="p">,</span>
+ <span class="n">prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">,</span>
+ <span class="n">delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">,</span>
+ <span class="n">aws_conn_id</span><span class="o">=</span><span class="n">aws_conn_id</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">dest_gcs_conn_id</span> <span class="o">=</span> <span class="n">dest_gcs_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">dest_gcs</span> <span class="o">=</span> <span class="n">dest_gcs</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">replace</span> <span class="o">=</span> <span class="n">replace</span>
+
+ <span class="k">if</span> <span class="n">dest_gcs</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_gcs_object_is_directory</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dest_gcs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'Destination Google Cloud Storage path is not a valid '</span>
+ <span class="s1">'"directory", define a path that ends with a slash "/" or '</span>
+ <span class="s1">'leave it empty for the root of the bucket.'</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'The destination Google Cloud Storage path '</span>
+ <span class="s1">'must end with a slash "/" or be empty.'</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="c1"># use the super method to list all the files in an S3 bucket/key</span>
+ <span class="n">files</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">S3ToGoogleCloudStorageOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">context</span><span class="p">)</span>
+
+ <span class="n">gcs_hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dest_gcs_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">replace</span><span class="p">:</span>
+ <span class="c1"># if we are not replacing -> list all files in the GCS bucket</span>
+ <span class="c1"># and only keep those files which are present in</span>
+ <span class="c1"># S3 and not in Google Cloud Storage</span>
+ <span class="n">bucket_name</span><span class="p">,</span> <span class="n">object_prefix</span> <span class="o">=</span> <span class="n">_parse_gcs_url</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dest_gcs</span><span class="p">)</span>
+ <span class="n">existing_files_prefixed</span> <span class="o">=</span> <span class="n">gcs_hook</span><span class="o">.</span><span class="n">list</span><span class="p">(</span>
+ <span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="n">object_prefix</span><span class="p">)</span>
+
+ <span class="n">existing_files</span> <span class="o">=</span> <span class="p">[]</span>
+
+ <span class="k">if</span> <span class="n">existing_files_prefixed</span><span class="p">:</span>
+ <span class="c1"># Remove the object prefix itself, an empty directory was found</span>
+ <span class="k">if</span> <span class="n">object_prefix</span> <span class="ow">in</span> <span class="n">existing_files_prefixed</span><span class="p">:</span>
+ <span class="n">existing_files_prefixed</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">object_prefix</span><span class="p">)</span>
+
+ <span class="c1"># Remove the object prefix from all object string paths</span>
+ <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">existing_files_prefixed</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">f</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">object_prefix</span><span class="p">):</span>
+ <span class="n">existing_files</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">f</span><span class="p">[</span><span class="nb">len</span><span class="p">(</span><span class="n">object_prefix</span><span class="p">):])</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">existing_files</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
+
+ <span class="n">files</span> <span class="o">=</span> <span class="nb">set</span><span class="p">(</span><span class="n">files</span><span class="p">)</span> <span class="o">-</span> <span class="nb">set</span><span class="p">(</span><span class="n">existing_files</span><span class="p">)</span>
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">files</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'</span><span class="si">{0}</span><span class="s1"> files are going to be synced: </span><span class="si">{1}</span><span class="s1">.'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="nb">len</span><span class="p">(</span><span class="n">files</span><span class="p">),</span> <span class="n">files</span><span class="p">))</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'There are no new files to sync. Have a nice day!'</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">files</span><span class="p">:</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">S3Hook</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span>
+
+ <span class="k">for</span> <span class="n">file</span> <span class="ow">in</span> <span class="n">files</span><span class="p">:</span>
+ <span class="c1"># GCS hook builds its own in-memory file so we have to create</span>
+ <span class="c1"># and pass the path</span>
+ <span class="n">file_object</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_key</span><span class="p">(</span><span class="n">file</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">)</span>
+ <span class="k">with</span> <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="n">mode</span><span class="o">=</span><span class="s1">'wb'</span><span class="p">,</span> <span class="n">delete</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
+ <span class="n">file_object</span><span class="o">.</span><span class="n">download_fileobj</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
+ <span class="n">f</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
+
+ <span class="n">dest_gcs_bucket</span><span class="p">,</span> <span class="n">dest_gcs_object_prefix</span> <span class="o">=</span> <span class="n">_parse_gcs_url</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">dest_gcs</span><span class="p">)</span>
+ <span class="c1"># There will always be a '/' before file because it is</span>
+ <span class="c1"># enforced at instantiation time</span>
+ <span class="n">dest_gcs_object</span> <span class="o">=</span> <span class="n">dest_gcs_object_prefix</span> <span class="o">+</span> <span class="n">file</span>
+
+ <span class="c1"># Sync is sequential and the hook already logs too much</span>
+ <span class="c1"># so skip this for now</span>
+ <span class="c1"># self.log.info(</span>
+ <span class="c1"># 'Saving file {0} from S3 bucket {1} in GCS bucket {2}'</span>
+ <span class="c1"># ' as object {3}'.format(file, self.bucket,</span>
+ <span class="c1"># dest_gcs_bucket,</span>
+ <span class="c1"># dest_gcs_object))</span>
+
+ <span class="n">gcs_hook</span><span class="o">.</span><span class="n">upload</span><span class="p">(</span><span class="n">dest_gcs_bucket</span><span class="p">,</span> <span class="n">dest_gcs_object</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s2">"All done, uploaded </span><span class="si">%d</span><span class="s2"> files to Google Cloud Storage"</span><span class="p">,</span>
+ <span class="nb">len</span><span class="p">(</span><span class="n">files</span><span class="p">))</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'In sync, no files needed to be uploaded to Google Cloud'</span>
+ <span class="s1">'Storage'</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">files</span>
+
+ <span class="c1"># Following functionality may be better suited in</span>
+ <span class="c1"># airflow/contrib/hooks/gcs_hook.py</span>
+ <span class="k">def</span> <span class="nf">_gcs_object_is_directory</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">object</span><span class="p">):</span>
+ <span class="n">bucket</span><span class="p">,</span> <span class="n">blob</span> <span class="o">=</span> <span class="n">_parse_gcs_url</span><span class="p">(</span><span class="nb">object</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="n">blob</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span> <span class="ow">or</span> <span class="n">blob</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s1">'/'</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/sftp_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/sftp_operator.html b/_modules/airflow/contrib/operators/sftp_operator.html
new file mode 100644
index 0000000..128db98
--- /dev/null
+++ b/_modules/airflow/contrib/operators/sftp_operator.html
@@ -0,0 +1,336 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.sftp_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.sftp_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.sftp_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.ssh_hook</span> <span class="k">import</span> <span class="n">SSHHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<span class="k">class</span> <span class="nc">SFTPOperation</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+ <span class="n">PUT</span> <span class="o">=</span> <span class="s1">'put'</span>
+ <span class="n">GET</span> <span class="o">=</span> <span class="s1">'get'</span>
+
+
+<div class="viewcode-block" id="SFTPOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.sftp_operator.SFTPOperator">[docs]</a><span class="k">class</span> <span class="nc">SFTPOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> SFTPOperator for transferring files from remote host to local or vice a versa.</span>
+<span class="sd"> This operator uses ssh_hook to open sftp trasport channel that serve as basis</span>
+<span class="sd"> for file transfer.</span>
+
+<span class="sd"> :param ssh_hook: predefined ssh_hook to use for remote execution</span>
+<span class="sd"> :type ssh_hook: :class:`SSHHook`</span>
+<span class="sd"> :param ssh_conn_id: connection id from airflow Connections</span>
+<span class="sd"> :type ssh_conn_id: str</span>
+<span class="sd"> :param remote_host: remote host to connect</span>
+<span class="sd"> :type remote_host: str</span>
+<span class="sd"> :param local_filepath: local file path to get or put</span>
+<span class="sd"> :type local_filepath: str</span>
+<span class="sd"> :param remote_filepath: remote file path to get or put</span>
+<span class="sd"> :type remote_filepath: str</span>
+<span class="sd"> :param operation: specify operation 'get' or 'put', defaults to get</span>
+<span class="sd"> :type get: bool</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'local_filepath'</span><span class="p">,</span> <span class="s1">'remote_filepath'</span><span class="p">)</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">ssh_hook</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">ssh_conn_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">remote_host</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">local_filepath</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">remote_filepath</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">operation</span><span class="o">=</span><span class="n">SFTPOperation</span><span class="o">.</span><span class="n">PUT</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">SFTPOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">ssh_hook</span> <span class="o">=</span> <span class="n">ssh_hook</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span> <span class="o">=</span> <span class="n">ssh_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span> <span class="o">=</span> <span class="n">remote_host</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">local_filepath</span> <span class="o">=</span> <span class="n">local_filepath</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">remote_filepath</span> <span class="o">=</span> <span class="n">remote_filepath</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">operation</span> <span class="o">=</span> <span class="n">operation</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">operation</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="n">SFTPOperation</span><span class="o">.</span><span class="n">GET</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">operation</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="n">SFTPOperation</span><span class="o">.</span><span class="n">PUT</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">"unsupported operation value </span><span class="si">{0}</span><span class="s2">, expected </span><span class="si">{1}</span><span class="s2"> or </span><span class="si">{2}</span><span class="s2">"</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">operation</span><span class="p">,</span> <span class="n">SFTPOperation</span><span class="o">.</span><span class="n">GET</span><span class="p">,</span> <span class="n">SFTPOperation</span><span class="o">.</span><span class="n">PUT</span><span class="p">))</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">file_msg</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_hook</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">ssh_hook</span> <span class="o">=</span> <span class="n">SSHHook</span><span class="p">(</span><span class="n">ssh_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_hook</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"can not operate without ssh_hook or ssh_conn_id"</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">ssh_hook</span><span class="o">.</span><span class="n">remote_host</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span>
+
+ <span class="n">ssh_client</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">sftp_client</span> <span class="o">=</span> <span class="n">ssh_client</span><span class="o">.</span><span class="n">open_sftp</span><span class="p">()</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">operation</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="n">SFTPOperation</span><span class="o">.</span><span class="n">GET</span><span class="p">:</span>
+ <span class="n">file_msg</span> <span class="o">=</span> <span class="s2">"from </span><span class="si">{0}</span><span class="s2"> to </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">remote_filepath</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">local_filepath</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Starting to transfer </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">file_msg</span><span class="p">)</span>
+ <span class="n">sftp_client</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">remote_filepath</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">local_filepath</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">file_msg</span> <span class="o">=</span> <span class="s2">"from </span><span class="si">{0}</span><span class="s2"> to </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">local_filepath</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">remote_filepath</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Starting to transfer file </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">file_msg</span><span class="p">)</span>
+ <span class="n">sftp_client</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">local_filepath</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">remote_filepath</span><span class="p">)</span>
+
+ <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Error while transferring </span><span class="si">{0}</span><span class="s2">, error: </span><span class="si">{1}</span><span class="s2">"</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">file_msg</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)))</span>
+
+ <span class="k">return</span> <span class="kc">None</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/slack_webhook_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/slack_webhook_operator.html b/_modules/airflow/contrib/operators/slack_webhook_operator.html
new file mode 100644
index 0000000..51eae15
--- /dev/null
+++ b/_modules/airflow/contrib/operators/slack_webhook_operator.html
@@ -0,0 +1,327 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.slack_webhook_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.slack_webhook_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.slack_webhook_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+<span class="kn">from</span> <span class="nn">airflow.operators.http_operator</span> <span class="k">import</span> <span class="n">SimpleHttpOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.slack_webhook_hook</span> <span class="k">import</span> <span class="n">SlackWebhookHook</span>
+
+
+<div class="viewcode-block" id="SlackWebhookOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.slack_webhook_operator.SlackWebhookOperator">[docs]</a><span class="k">class</span> <span class="nc">SlackWebhookOperator</span><span class="p">(</span><span class="n">SimpleHttpOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> This operator allows you to post messages to Slack using incoming webhooks.</span>
+<span class="sd"> Takes both Slack webhook token directly and connection that has Slack webhook token.</span>
+<span class="sd"> If both supplied, Slack webhook token will be used.</span>
+
+<span class="sd"> Each Slack webhook token can be pre-configured to use a specific channel, username and</span>
+<span class="sd"> icon. You can override these defaults in this hook.</span>
+
+<span class="sd"> :param conn_id: connection that has Slack webhook token in the extra field</span>
+<span class="sd"> :type conn_id: str</span>
+<span class="sd"> :param webhook_token: Slack webhook token</span>
+<span class="sd"> :type webhook_token: str</span>
+<span class="sd"> :param message: The message you want to send on Slack</span>
+<span class="sd"> :type message: str</span>
+<span class="sd"> :param channel: The channel the message should be posted to</span>
+<span class="sd"> :type channel: str</span>
+<span class="sd"> :param username: The username to post to slack with</span>
+<span class="sd"> :type username: str</span>
+<span class="sd"> :param icon_emoji: The emoji to use as icon for the user posting to Slack</span>
+<span class="sd"> :type icon_emoji: str</span>
+<span class="sd"> :param link_names: Whether or not to find and link channel and usernames in your</span>
+<span class="sd"> message</span>
+<span class="sd"> :type link_names: bool</span>
+<span class="sd"> :param proxy: Proxy to use to make the Slack webhook call</span>
+<span class="sd"> :type proxy: str</span>
+<span class="sd"> """</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">http_conn_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">webhook_token</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">message</span><span class="o">=</span><span class="s2">""</span><span class="p">,</span>
+ <span class="n">channel</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">username</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">icon_emoji</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">link_names</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">proxy</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">SlackWebhookOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">endpoint</span><span class="o">=</span><span class="n">webhook_token</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">http_conn_id</span> <span class="o">=</span> <span class="n">http_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">webhook_token</span> <span class="o">=</span> <span class="n">webhook_token</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">message</span> <span class="o">=</span> <span class="n">message</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">channel</span> <span class="o">=</span> <span class="n">channel</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="o">=</span> <span class="n">username</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">icon_emoji</span> <span class="o">=</span> <span class="n">icon_emoji</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">link_names</span> <span class="o">=</span> <span class="n">link_names</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">proxy</span> <span class="o">=</span> <span class="n">proxy</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="kc">None</span>
+
+<div class="viewcode-block" id="SlackWebhookOperator.execute"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.slack_webhook_operator.SlackWebhookOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Call the SparkSqlHook to run the provided sql query</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">SlackWebhookHook</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">http_conn_id</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">webhook_token</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">message</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">channel</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">icon_emoji</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">link_names</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">proxy</span>
+ <span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[21/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/sensors/emr_job_flow_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/sensors/emr_job_flow_sensor.html b/_modules/airflow/contrib/sensors/emr_job_flow_sensor.html
new file mode 100644
index 0000000..4fcbbc9
--- /dev/null
+++ b/_modules/airflow/contrib/sensors/emr_job_flow_sensor.html
@@ -0,0 +1,288 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.sensors.emr_job_flow_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.sensors.emr_job_flow_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.sensors.emr_job_flow_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.emr_hook</span> <span class="k">import</span> <span class="n">EmrHook</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.sensors.emr_base_sensor</span> <span class="k">import</span> <span class="n">EmrBaseSensor</span>
+<span class="kn">from</span> <span class="nn">airflow.utils</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="EmrJobFlowSensor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.sensors.emr_job_flow_sensor.EmrJobFlowSensor">[docs]</a><span class="k">class</span> <span class="nc">EmrJobFlowSensor</span><span class="p">(</span><span class="n">EmrBaseSensor</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Asks for the state of the JobFlow until it reaches a terminal state.</span>
+<span class="sd"> If it fails the sensor errors, failing the task.</span>
+
+<span class="sd"> :param job_flow_id: job_flow_id to check the state of</span>
+<span class="sd"> :type job_flow_id: string</span>
+<span class="sd"> """</span>
+
+ <span class="n">NON_TERMINAL_STATES</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'STARTING'</span><span class="p">,</span> <span class="s1">'BOOTSTRAPPING'</span><span class="p">,</span> <span class="s1">'RUNNING'</span><span class="p">,</span> <span class="s1">'WAITING'</span><span class="p">,</span> <span class="s1">'TERMINATING'</span><span class="p">]</span>
+ <span class="n">FAILED_STATE</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'TERMINATED_WITH_ERRORS'</span><span class="p">]</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'job_flow_id'</span><span class="p">]</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">()</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">job_flow_id</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">EmrJobFlowSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job_flow_id</span> <span class="o">=</span> <span class="n">job_flow_id</span>
+
+ <span class="k">def</span> <span class="nf">get_emr_response</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="n">emr</span> <span class="o">=</span> <span class="n">EmrHook</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Poking cluster </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_flow_id</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">emr</span><span class="o">.</span><span class="n">describe_cluster</span><span class="p">(</span><span class="n">ClusterId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">job_flow_id</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">state_from_response</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">response</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">response</span><span class="p">[</span><span class="s1">'Cluster'</span><span class="p">][</span><span class="s1">'Status'</span><span class="p">][</span><span class="s1">'State'</span><span class="p">]</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/sensors/emr_step_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/sensors/emr_step_sensor.html b/_modules/airflow/contrib/sensors/emr_step_sensor.html
new file mode 100644
index 0000000..a5a3e3e
--- /dev/null
+++ b/_modules/airflow/contrib/sensors/emr_step_sensor.html
@@ -0,0 +1,292 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.sensors.emr_step_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.sensors.emr_step_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.sensors.emr_step_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.emr_hook</span> <span class="k">import</span> <span class="n">EmrHook</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.sensors.emr_base_sensor</span> <span class="k">import</span> <span class="n">EmrBaseSensor</span>
+<span class="kn">from</span> <span class="nn">airflow.utils</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="EmrStepSensor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.sensors.emr_step_sensor.EmrStepSensor">[docs]</a><span class="k">class</span> <span class="nc">EmrStepSensor</span><span class="p">(</span><span class="n">EmrBaseSensor</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Asks for the state of the step until it reaches a terminal state.</span>
+<span class="sd"> If it fails the sensor errors, failing the task.</span>
+
+<span class="sd"> :param job_flow_id: job_flow_id which contains the step check the state of</span>
+<span class="sd"> :type job_flow_id: string</span>
+<span class="sd"> :param step_id: step to check the state of</span>
+<span class="sd"> :type step_id: string</span>
+<span class="sd"> """</span>
+
+ <span class="n">NON_TERMINAL_STATES</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'PENDING'</span><span class="p">,</span> <span class="s1">'RUNNING'</span><span class="p">,</span> <span class="s1">'CONTINUE'</span><span class="p">]</span>
+ <span class="n">FAILED_STATE</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'CANCELLED'</span><span class="p">,</span> <span class="s1">'FAILED'</span><span class="p">]</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'job_flow_id'</span><span class="p">,</span> <span class="s1">'step_id'</span><span class="p">]</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">()</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">job_flow_id</span><span class="p">,</span>
+ <span class="n">step_id</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">EmrStepSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job_flow_id</span> <span class="o">=</span> <span class="n">job_flow_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">step_id</span> <span class="o">=</span> <span class="n">step_id</span>
+
+ <span class="k">def</span> <span class="nf">get_emr_response</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="n">emr</span> <span class="o">=</span> <span class="n">EmrHook</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Poking step </span><span class="si">%s</span><span class="s1"> on cluster </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">step_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_flow_id</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">emr</span><span class="o">.</span><span class="n">describe_step</span><span class="p">(</span><span class="n">ClusterId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">job_flow_id</span><span class="p">,</span> <span class="n">StepId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">step_id</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">state_from_response</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">response</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">response</span><span class="p">[</span><span class="s1">'Step'</span><span class="p">][</span><span class="s1">'Status'</span><span class="p">][</span><span class="s1">'State'</span><span class="p">]</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/sensors/file_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/sensors/file_sensor.html b/_modules/airflow/contrib/sensors/file_sensor.html
new file mode 100644
index 0000000..9d87886
--- /dev/null
+++ b/_modules/airflow/contrib/sensors/file_sensor.html
@@ -0,0 +1,306 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.sensors.file_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.sensors.file_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.sensors.file_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+
+<span class="kn">import</span> <span class="nn">os</span>
+<span class="kn">import</span> <span class="nn">stat</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.fs_hook</span> <span class="k">import</span> <span class="n">FSHook</span>
+<span class="kn">from</span> <span class="nn">airflow.sensors.base_sensor_operator</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="FileSensor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.sensors.file_sensor.FileSensor">[docs]</a><span class="k">class</span> <span class="nc">FileSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Waits for a file or folder to land in a filesystem.</span>
+
+<span class="sd"> If the path given is a directory then this sensor will only return true if</span>
+<span class="sd"> any files exist inside it (either directly, or within a subdirectory)</span>
+
+<span class="sd"> :param fs_conn_id: reference to the File (path)</span>
+<span class="sd"> connection id</span>
+<span class="sd"> :type fs_conn_id: string</span>
+<span class="sd"> :param filepath: File or folder name (relative to</span>
+<span class="sd"> the base path set within the connection)</span>
+<span class="sd"> :type fs_conn_id: string</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'filepath'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#91818a'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">filepath</span><span class="p">,</span>
+ <span class="n">fs_conn_id</span><span class="o">=</span><span class="s1">'fs_default2'</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">FileSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">filepath</span> <span class="o">=</span> <span class="n">filepath</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">fs_conn_id</span> <span class="o">=</span> <span class="n">fs_conn_id</span>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">FSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fs_conn_id</span><span class="p">)</span>
+ <span class="n">basepath</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_path</span><span class="p">()</span>
+ <span class="n">full_path</span> <span class="o">=</span> <span class="s2">"/"</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">basepath</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="p">])</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Poking for file </span><span class="si">{full_path}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">stat</span><span class="o">.</span><span class="n">S_ISDIR</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">full_path</span><span class="p">)</span><span class="o">.</span><span class="n">st_mode</span><span class="p">):</span>
+ <span class="k">for</span> <span class="n">root</span><span class="p">,</span> <span class="n">dirs</span><span class="p">,</span> <span class="n">files</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">walk</span><span class="p">(</span><span class="n">full_path</span><span class="p">):</span>
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">files</span><span class="p">):</span>
+ <span class="k">return</span> <span class="kc">True</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="c1"># full_path was a file directly</span>
+ <span class="k">return</span> <span class="kc">True</span>
+ <span class="k">except</span> <span class="ne">OSError</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">False</span>
+ <span class="k">return</span> <span class="kc">False</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/sensors/ftp_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/sensors/ftp_sensor.html b/_modules/airflow/contrib/sensors/ftp_sensor.html
new file mode 100644
index 0000000..d6d4924
--- /dev/null
+++ b/_modules/airflow/contrib/sensors/ftp_sensor.html
@@ -0,0 +1,302 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.sensors.ftp_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.sensors.ftp_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.sensors.ftp_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">import</span> <span class="nn">ftplib</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.ftp_hook</span> <span class="k">import</span> <span class="n">FTPHook</span><span class="p">,</span> <span class="n">FTPSHook</span>
+<span class="kn">from</span> <span class="nn">airflow.sensors.base_sensor_operator</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="FTPSensor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.sensors.ftp_sensor.FTPSensor">[docs]</a><span class="k">class</span> <span class="nc">FTPSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Waits for a file or directory to be present on FTP.</span>
+
+<span class="sd"> :param path: Remote file or directory path</span>
+<span class="sd"> :type path: str</span>
+<span class="sd"> :param ftp_conn_id: The connection to run the sensor against</span>
+<span class="sd"> :type ftp_conn_id: str</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'path'</span><span class="p">,)</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">ftp_conn_id</span><span class="o">=</span><span class="s1">'ftp_default'</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">FTPSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">path</span> <span class="o">=</span> <span class="n">path</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">ftp_conn_id</span> <span class="o">=</span> <span class="n">ftp_conn_id</span>
+
+ <span class="k">def</span> <span class="nf">_create_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""Return connection hook."""</span>
+ <span class="k">return</span> <span class="n">FTPHook</span><span class="p">(</span><span class="n">ftp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">ftp_conn_id</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">_create_hook</span><span class="p">()</span> <span class="k">as</span> <span class="n">hook</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Poking for </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">path</span><span class="p">)</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">hook</span><span class="o">.</span><span class="n">get_mod_time</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">path</span><span class="p">)</span>
+ <span class="k">except</span> <span class="n">ftplib</span><span class="o">.</span><span class="n">error_perm</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="n">error</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">error</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">!=</span> <span class="s2">"Can't check for file existence"</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">e</span>
+
+ <span class="k">return</span> <span class="kc">False</span>
+
+ <span class="k">return</span> <span class="kc">True</span></div>
+
+
+<div class="viewcode-block" id="FTPSSensor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.sensors.ftp_sensor.FTPSSensor">[docs]</a><span class="k">class</span> <span class="nc">FTPSSensor</span><span class="p">(</span><span class="n">FTPSensor</span><span class="p">):</span>
+ <span class="sd">"""Waits for a file or directory to be present on FTP over SSL."""</span>
+ <span class="k">def</span> <span class="nf">_create_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""Return connection hook."""</span>
+ <span class="k">return</span> <span class="n">FTPSHook</span><span class="p">(</span><span class="n">ftp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">ftp_conn_id</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/sensors/gcs_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/sensors/gcs_sensor.html b/_modules/airflow/contrib/sensors/gcs_sensor.html
new file mode 100644
index 0000000..aee0eda
--- /dev/null
+++ b/_modules/airflow/contrib/sensors/gcs_sensor.html
@@ -0,0 +1,408 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.sensors.gcs_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.sensors.gcs_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.sensors.gcs_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcs_hook</span> <span class="k">import</span> <span class="n">GoogleCloudStorageHook</span>
+<span class="kn">from</span> <span class="nn">airflow.sensors.base_sensor_operator</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="GoogleCloudStorageObjectSensor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.sensors.gcs_sensor.GoogleCloudStorageObjectSensor">[docs]</a><span class="k">class</span> <span class="nc">GoogleCloudStorageObjectSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Checks for the existence of a file in Google Cloud Storage.</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bucket'</span><span class="p">,</span> <span class="s1">'object'</span><span class="p">)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f0eee4'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">bucket</span><span class="p">,</span>
+ <span class="nb">object</span><span class="p">,</span> <span class="c1"># pylint:disable=redefined-builtin</span>
+ <span class="n">google_cloud_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Create a new GoogleCloudStorageObjectSensor.</span>
+
+<span class="sd"> :param bucket: The Google cloud storage bucket where the object is.</span>
+<span class="sd"> :type bucket: string</span>
+<span class="sd"> :param object: The name of the object to check in the Google cloud</span>
+<span class="sd"> storage bucket.</span>
+<span class="sd"> :type object: string</span>
+<span class="sd"> :param google_cloud_storage_conn_id: The connection ID to use when</span>
+<span class="sd"> connecting to Google cloud storage.</span>
+<span class="sd"> :type google_cloud_storage_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request must have domain-wide delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> """</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">GoogleCloudStorageObjectSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">object</span> <span class="o">=</span> <span class="nb">object</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_conn_id</span> <span class="o">=</span> <span class="n">google_cloud_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Sensor checks existence of : </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">object</span><span class="p">)</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">object</span><span class="p">)</span></div>
+
+
+<span class="k">def</span> <span class="nf">ts_function</span><span class="p">(</span><span class="n">context</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Default callback for the GoogleCloudStorageObjectUpdatedSensor. The default</span>
+<span class="sd"> behaviour is check for the object being updated after execution_date +</span>
+<span class="sd"> schedule_interval.</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="n">context</span><span class="p">[</span><span class="s1">'execution_date'</span><span class="p">]</span> <span class="o">+</span> <span class="n">context</span><span class="p">[</span><span class="s1">'dag'</span><span class="p">]</span><span class="o">.</span><span class="n">schedule_interval</span>
+
+
+<div class="viewcode-block" id="GoogleCloudStorageObjectUpdatedSensor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.sensors.gcs_sensor.GoogleCloudStorageObjectUpdatedSensor">[docs]</a><span class="k">class</span> <span class="nc">GoogleCloudStorageObjectUpdatedSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Checks if an object is updated in Google Cloud Storage.</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bucket'</span><span class="p">,</span> <span class="s1">'object'</span><span class="p">)</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f0eee4'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">bucket</span><span class="p">,</span>
+ <span class="nb">object</span><span class="p">,</span> <span class="c1"># pylint:disable=redefined-builtin</span>
+ <span class="n">ts_func</span><span class="o">=</span><span class="n">ts_function</span><span class="p">,</span>
+ <span class="n">google_cloud_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Create a new GoogleCloudStorageObjectUpdatedSensor.</span>
+
+<span class="sd"> :param bucket: The Google cloud storage bucket where the object is.</span>
+<span class="sd"> :type bucket: string</span>
+<span class="sd"> :param object: The name of the object to download in the Google cloud</span>
+<span class="sd"> storage bucket.</span>
+<span class="sd"> :type object: string</span>
+<span class="sd"> :param ts_func: Callback for defining the update condition. The default callback</span>
+<span class="sd"> returns execution_date + schedule_interval. The callback takes the context</span>
+<span class="sd"> as parameter.</span>
+<span class="sd"> :type ts_func: function</span>
+<span class="sd"> :param google_cloud_storage_conn_id: The connection ID to use when</span>
+<span class="sd"> connecting to Google cloud storage.</span>
+<span class="sd"> :type google_cloud_storage_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request must have domain-wide</span>
+<span class="sd"> delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> """</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">GoogleCloudStorageObjectUpdatedSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">object</span> <span class="o">=</span> <span class="nb">object</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">ts_func</span> <span class="o">=</span> <span class="n">ts_func</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_conn_id</span> <span class="o">=</span> <span class="n">google_cloud_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Sensor checks existence of : </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">object</span><span class="p">)</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">is_updated_after</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">object</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ts_func</span><span class="p">(</span><span class="n">context</span><span class="p">))</span></div>
+
+
+<div class="viewcode-block" id="GoogleCloudStoragePrefixSensor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.sensors.gcs_sensor.GoogleCloudStoragePrefixSensor">[docs]</a><span class="k">class</span> <span class="nc">GoogleCloudStoragePrefixSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Checks for the existence of a files at prefix in Google Cloud Storage bucket.</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bucket'</span><span class="p">,</span> <span class="s1">'prefix'</span><span class="p">)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f0eee4'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">bucket</span><span class="p">,</span>
+ <span class="n">prefix</span><span class="p">,</span>
+ <span class="n">google_cloud_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Create a new GoogleCloudStorageObjectSensor.</span>
+
+<span class="sd"> :param bucket: The Google cloud storage bucket where the object is.</span>
+<span class="sd"> :type bucket: string</span>
+<span class="sd"> :param prefix: The name of the prefix to check in the Google cloud</span>
+<span class="sd"> storage bucket.</span>
+<span class="sd"> :type prefix: string</span>
+<span class="sd"> :param google_cloud_storage_conn_id: The connection ID to use when</span>
+<span class="sd"> connecting to Google cloud storage.</span>
+<span class="sd"> :type google_cloud_storage_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request must have domain-wide delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> """</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">GoogleCloudStoragePrefixSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">prefix</span> <span class="o">=</span> <span class="n">prefix</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_conn_id</span> <span class="o">=</span> <span class="n">google_cloud_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Sensor checks existence of objects: </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">prefix</span><span class="p">)</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+ <span class="k">return</span> <span class="nb">bool</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">prefix</span><span class="p">))</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[15/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/hooks/oracle_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/hooks/oracle_hook.html b/_modules/airflow/hooks/oracle_hook.html
new file mode 100644
index 0000000..3683348
--- /dev/null
+++ b/_modules/airflow/hooks/oracle_hook.html
@@ -0,0 +1,382 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.hooks.oracle_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.hooks.oracle_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.hooks.oracle_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">cx_Oracle</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="k">import</span> <span class="n">DbApiHook</span>
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
+<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="k">import</span> <span class="n">basestring</span>
+<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span>
+<span class="kn">import</span> <span class="nn">numpy</span>
+
+
+<div class="viewcode-block" id="OracleHook"><a class="viewcode-back" href="../../../code.html#airflow.hooks.oracle_hook.OracleHook">[docs]</a><span class="k">class</span> <span class="nc">OracleHook</span><span class="p">(</span><span class="n">DbApiHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with Oracle SQL.</span>
+<span class="sd"> """</span>
+ <span class="n">conn_name_attr</span> <span class="o">=</span> <span class="s1">'oracle_conn_id'</span>
+ <span class="n">default_conn_name</span> <span class="o">=</span> <span class="s1">'oracle_default'</span>
+ <span class="n">supports_autocommit</span> <span class="o">=</span> <span class="kc">False</span>
+
+<div class="viewcode-block" id="OracleHook.get_conn"><a class="viewcode-back" href="../../../code.html#airflow.hooks.oracle_hook.OracleHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns a oracle connection object</span>
+<span class="sd"> Optional parameters for using a custom DSN connection (instead of using a server alias from tnsnames.ora)</span>
+<span class="sd"> The dsn (data source name) is the TNS entry (from the Oracle names server or tnsnames.ora file)</span>
+<span class="sd"> or is a string like the one returned from makedsn().</span>
+
+<span class="sd"> :param dsn: the host address for the Oracle server</span>
+<span class="sd"> :param service_name: the db_unique_name of the database that you are connecting to (CONNECT_DATA part of TNS)</span>
+<span class="sd"> You can set these parameters in the extra fields of your connection</span>
+<span class="sd"> as in ``{ "dsn":"some.host.address" , "service_name":"some.service.name" }``</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">oracle_conn_id</span><span class="p">)</span>
+ <span class="n">dsn</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'dsn'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="n">sid</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'sid'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="n">mod</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'module'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+
+ <span class="n">service_name</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'service_name'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">dsn</span> <span class="ow">and</span> <span class="n">sid</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">service_name</span><span class="p">:</span>
+ <span class="n">dsn</span> <span class="o">=</span> <span class="n">cx_Oracle</span><span class="o">.</span><span class="n">makedsn</span><span class="p">(</span><span class="n">dsn</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">,</span> <span class="n">sid</span><span class="p">)</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="n">cx_Oracle</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span> <span class="n">dsn</span><span class="o">=</span><span class="n">dsn</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="n">dsn</span> <span class="ow">and</span> <span class="n">service_name</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">sid</span><span class="p">:</span>
+ <span class="n">dsn</span> <span class="o">=</span> <span class="n">cx_Oracle</span><span class="o">.</span><span class="n">makedsn</span><span class="p">(</span><span class="n">dsn</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">,</span> <span class="n">service_name</span><span class="o">=</span><span class="n">service_name</span><span class="p">)</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="n">cx_Oracle</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span> <span class="n">dsn</span><span class="o">=</span><span class="n">dsn</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="n">cx_Oracle</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">mod</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">module</span> <span class="o">=</span> <span class="n">mod</span>
+
+ <span class="k">return</span> <span class="n">conn</span></div>
+
+<div class="viewcode-block" id="OracleHook.insert_rows"><a class="viewcode-back" href="../../../code.html#airflow.hooks.oracle_hook.OracleHook.insert_rows">[docs]</a> <span class="k">def</span> <span class="nf">insert_rows</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">rows</span><span class="p">,</span> <span class="n">target_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">commit_every</span><span class="o">=</span><span class="mi">1000</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> A generic way to insert a set of tuples into a table,</span>
+<span class="sd"> the whole set of inserts is treated as one transaction</span>
+<span class="sd"> Changes from standard DbApiHook implementation:</span>
+<span class="sd"> - Oracle SQL queries in cx_Oracle can not be terminated with a semicolon (';')</span>
+<span class="sd"> - Replace NaN values with NULL using numpy.nan_to_num (not using is_nan() because of input types error for strings)</span>
+<span class="sd"> - Coerce datetime cells to Oracle DATETIME format during insert</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">target_fields</span><span class="p">:</span>
+ <span class="n">target_fields</span> <span class="o">=</span> <span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">target_fields</span><span class="p">)</span>
+ <span class="n">target_fields</span> <span class="o">=</span> <span class="s1">'(</span><span class="si">{}</span><span class="s1">)'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">target_fields</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">target_fields</span> <span class="o">=</span> <span class="s1">''</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">cur</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">supports_autocommit</span><span class="p">:</span>
+ <span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="s1">'SET autocommit = 0'</span><span class="p">)</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
+ <span class="n">i</span> <span class="o">=</span> <span class="mi">0</span>
+ <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">rows</span><span class="p">:</span>
+ <span class="n">i</span> <span class="o">+=</span> <span class="mi">1</span>
+ <span class="n">l</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="k">for</span> <span class="n">cell</span> <span class="ow">in</span> <span class="n">row</span><span class="p">:</span>
+ <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">cell</span><span class="p">,</span> <span class="n">basestring</span><span class="p">):</span>
+ <span class="n">l</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"'"</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">cell</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"'"</span><span class="p">,</span> <span class="s2">"''"</span><span class="p">)</span> <span class="o">+</span> <span class="s2">"'"</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="n">cell</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">l</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s1">'NULL'</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="nb">type</span><span class="p">(</span><span class="n">cell</span><span class="p">)</span> <span class="o">==</span> <span class="nb">float</span> <span class="ow">and</span> <span class="n">numpy</span><span class="o">.</span><span class="n">isnan</span><span class="p">(</span><span class="n">cell</span><span class="p">):</span> <span class="c1"># coerce numpy NaN to NULL</span>
+ <span class="n">l</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s1">'NULL'</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">cell</span><span class="p">,</span> <span class="n">numpy</span><span class="o">.</span><span class="n">datetime64</span><span class="p">):</span>
+ <span class="n">l</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"'"</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">cell</span><span class="p">)</span> <span class="o">+</span> <span class="s2">"'"</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">cell</span><span class="p">,</span> <span class="n">datetime</span><span class="p">):</span>
+ <span class="n">l</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"to_date('"</span> <span class="o">+</span> <span class="n">cell</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s1">'%Y-%m-</span><span class="si">%d</span><span class="s1"> %H:%M:%S'</span><span class="p">)</span> <span class="o">+</span> <span class="s2">"','YYYY-MM-DD HH24:MI:SS')"</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">l</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">cell</span><span class="p">))</span>
+ <span class="n">values</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">l</span><span class="p">)</span>
+ <span class="n">sql</span> <span class="o">=</span> <span class="s1">'INSERT /*+ APPEND */ INTO </span><span class="si">{0}</span><span class="s1"> </span><span class="si">{1}</span><span class="s1"> VALUES (</span><span class="si">{2}</span><span class="s1">)'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">table</span><span class="p">,</span> <span class="n">target_fields</span><span class="p">,</span> <span class="s1">','</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">values</span><span class="p">))</span>
+ <span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">i</span> <span class="o">%</span> <span class="n">commit_every</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Loaded </span><span class="si">{i}</span><span class="s1"> into </span><span class="si">{table}</span><span class="s1"> rows so far'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
+ <span class="n">cur</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Done loading. Loaded a total of </span><span class="si">{i}</span><span class="s1"> rows'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span></div>
+
+<div class="viewcode-block" id="OracleHook.bulk_insert_rows"><a class="viewcode-back" href="../../../code.html#airflow.hooks.oracle_hook.OracleHook.bulk_insert_rows">[docs]</a> <span class="k">def</span> <span class="nf">bulk_insert_rows</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">rows</span><span class="p">,</span> <span class="n">target_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">commit_every</span><span class="o">=</span><span class="mi">5000</span><span class="p">):</span>
+ <span class="sd">"""A performant bulk insert for cx_Oracle that uses prepared statements via `executemany()`.</span>
+<span class="sd"> For best performance, pass in `rows` as an iterator.</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
+ <span class="n">values</span> <span class="o">=</span> <span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="s1">':</span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">i</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">target_fields</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span><span class="p">))</span>
+ <span class="n">prepared_stm</span> <span class="o">=</span> <span class="s1">'insert into </span><span class="si">{tablename}</span><span class="s1"> (</span><span class="si">{columns}</span><span class="s1">) values (</span><span class="si">{values}</span><span class="s1">)'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">tablename</span><span class="o">=</span><span class="n">table</span><span class="p">,</span>
+ <span class="n">columns</span><span class="o">=</span><span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">target_fields</span><span class="p">),</span>
+ <span class="n">values</span><span class="o">=</span><span class="n">values</span><span class="p">,</span>
+ <span class="p">)</span>
+ <span class="n">row_count</span> <span class="o">=</span> <span class="mi">0</span>
+ <span class="c1"># Chunk the rows</span>
+ <span class="n">row_chunk</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">rows</span><span class="p">:</span>
+ <span class="n">row_chunk</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">row</span><span class="p">)</span>
+ <span class="n">row_count</span> <span class="o">+=</span> <span class="mi">1</span>
+ <span class="k">if</span> <span class="n">row_count</span> <span class="o">%</span> <span class="n">commit_every</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="n">cursor</span><span class="o">.</span><span class="n">prepare</span><span class="p">(</span><span class="n">prepared_stm</span><span class="p">)</span>
+ <span class="n">cursor</span><span class="o">.</span><span class="n">executemany</span><span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="n">row_chunk</span><span class="p">)</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'[</span><span class="si">%s</span><span class="s1">] inserted </span><span class="si">%s</span><span class="s1"> rows'</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">row_count</span><span class="p">)</span>
+ <span class="c1"># Empty chunk</span>
+ <span class="n">row_chunk</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="c1"># Commit the leftover chunk</span>
+ <span class="n">cursor</span><span class="o">.</span><span class="n">prepare</span><span class="p">(</span><span class="n">prepared_stm</span><span class="p">)</span>
+ <span class="n">cursor</span><span class="o">.</span><span class="n">executemany</span><span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="n">row_chunk</span><span class="p">)</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'[</span><span class="si">%s</span><span class="s1">] inserted </span><span class="si">%s</span><span class="s1"> rows'</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">row_count</span><span class="p">)</span>
+ <span class="n">cursor</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/hooks/pig_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/hooks/pig_hook.html b/_modules/airflow/hooks/pig_hook.html
new file mode 100644
index 0000000..05fac3c
--- /dev/null
+++ b/_modules/airflow/hooks/pig_hook.html
@@ -0,0 +1,329 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.hooks.pig_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.hooks.pig_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.hooks.pig_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="k">import</span> <span class="n">print_function</span>
+<span class="kn">import</span> <span class="nn">subprocess</span>
+<span class="kn">from</span> <span class="nn">tempfile</span> <span class="k">import</span> <span class="n">NamedTemporaryFile</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.file</span> <span class="k">import</span> <span class="n">TemporaryDirectory</span>
+
+
+<div class="viewcode-block" id="PigCliHook"><a class="viewcode-back" href="../../../code.html#airflow.hooks.pig_hook.PigCliHook">[docs]</a><span class="k">class</span> <span class="nc">PigCliHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Simple wrapper around the pig CLI.</span>
+
+<span class="sd"> Note that you can also set default pig CLI properties using the</span>
+<span class="sd"> ``pig_properties`` to be used in your connection as in</span>
+<span class="sd"> ``{"pig_properties": "-Dpig.tmpfilecompression=true"}``</span>
+
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">pig_cli_conn_id</span><span class="o">=</span><span class="s2">"pig_cli_default"</span><span class="p">):</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">pig_cli_conn_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">pig_properties</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'pig_properties'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="n">conn</span>
+
+<div class="viewcode-block" id="PigCliHook.run_cli"><a class="viewcode-back" href="../../../code.html#airflow.hooks.pig_hook.PigCliHook.run_cli">[docs]</a> <span class="k">def</span> <span class="nf">run_cli</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">pig</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Run an pig script using the pig cli</span>
+
+<span class="sd"> >>> ph = PigCliHook()</span>
+<span class="sd"> >>> result = ph.run_cli("ls /;")</span>
+<span class="sd"> >>> ("hdfs://" in result)</span>
+<span class="sd"> True</span>
+<span class="sd"> """</span>
+
+ <span class="k">with</span> <span class="n">TemporaryDirectory</span><span class="p">(</span><span class="n">prefix</span><span class="o">=</span><span class="s1">'airflow_pigop_'</span><span class="p">)</span> <span class="k">as</span> <span class="n">tmp_dir</span><span class="p">:</span>
+ <span class="k">with</span> <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="nb">dir</span><span class="o">=</span><span class="n">tmp_dir</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
+ <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">pig</span><span class="p">)</span>
+ <span class="n">f</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
+ <span class="n">fname</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">name</span>
+ <span class="n">pig_bin</span> <span class="o">=</span> <span class="s1">'pig'</span>
+ <span class="n">cmd_extra</span> <span class="o">=</span> <span class="p">[]</span>
+
+ <span class="n">pig_cmd</span> <span class="o">=</span> <span class="p">[</span><span class="n">pig_bin</span><span class="p">,</span> <span class="s1">'-f'</span><span class="p">,</span> <span class="n">fname</span><span class="p">]</span> <span class="o">+</span> <span class="n">cmd_extra</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">pig_properties</span><span class="p">:</span>
+ <span class="n">pig_properties_list</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">pig_properties</span><span class="o">.</span><span class="n">split</span><span class="p">()</span>
+ <span class="n">pig_cmd</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">pig_properties_list</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">verbose</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">" "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">pig_cmd</span><span class="p">))</span>
+ <span class="n">sp</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">Popen</span><span class="p">(</span>
+ <span class="n">pig_cmd</span><span class="p">,</span>
+ <span class="n">stdout</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span>
+ <span class="n">stderr</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">STDOUT</span><span class="p">,</span>
+ <span class="n">cwd</span><span class="o">=</span><span class="n">tmp_dir</span><span class="p">,</span>
+ <span class="n">close_fds</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sp</span> <span class="o">=</span> <span class="n">sp</span>
+ <span class="n">stdout</span> <span class="o">=</span> <span class="s1">''</span>
+ <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="nb">iter</span><span class="p">(</span><span class="n">sp</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">readline</span><span class="p">,</span> <span class="s1">''</span><span class="p">):</span>
+ <span class="n">stdout</span> <span class="o">+=</span> <span class="n">line</span>
+ <span class="k">if</span> <span class="n">verbose</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">line</span><span class="o">.</span><span class="n">strip</span><span class="p">())</span>
+ <span class="n">sp</span><span class="o">.</span><span class="n">wait</span><span class="p">()</span>
+
+ <span class="k">if</span> <span class="n">sp</span><span class="o">.</span><span class="n">returncode</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="n">stdout</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">stdout</span></div>
+
+ <span class="k">def</span> <span class="nf">kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">'sp'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">sp</span><span class="o">.</span><span class="n">poll</span><span class="p">()</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="nb">print</span><span class="p">(</span><span class="s2">"Killing the Pig job"</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sp</span><span class="o">.</span><span class="n">kill</span><span class="p">()</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/hooks/postgres_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/hooks/postgres_hook.html b/_modules/airflow/hooks/postgres_hook.html
new file mode 100644
index 0000000..d912cc2
--- /dev/null
+++ b/_modules/airflow/hooks/postgres_hook.html
@@ -0,0 +1,323 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.hooks.postgres_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.hooks.postgres_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.hooks.postgres_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">psycopg2</span>
+<span class="kn">import</span> <span class="nn">psycopg2.extensions</span>
+<span class="kn">from</span> <span class="nn">contextlib</span> <span class="k">import</span> <span class="n">closing</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="k">import</span> <span class="n">DbApiHook</span>
+
+
+<div class="viewcode-block" id="PostgresHook"><a class="viewcode-back" href="../../../code.html#airflow.hooks.postgres_hook.PostgresHook">[docs]</a><span class="k">class</span> <span class="nc">PostgresHook</span><span class="p">(</span><span class="n">DbApiHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with Postgres.</span>
+<span class="sd"> You can specify ssl parameters in the extra field of your connection</span>
+<span class="sd"> as ``{"sslmode": "require", "sslcert": "/path/to/cert.pem", etc}``.</span>
+
+<span class="sd"> Note: For Redshift, use keepalives_idle in the extra connection parameters</span>
+<span class="sd"> and set it to less than 300 seconds.</span>
+<span class="sd"> """</span>
+ <span class="n">conn_name_attr</span> <span class="o">=</span> <span class="s1">'postgres_conn_id'</span>
+ <span class="n">default_conn_name</span> <span class="o">=</span> <span class="s1">'postgres_default'</span>
+ <span class="n">supports_autocommit</span> <span class="o">=</span> <span class="kc">True</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">PostgresHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s2">"schema"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">postgres_conn_id</span><span class="p">)</span>
+ <span class="n">conn_args</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
+ <span class="n">host</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">,</span>
+ <span class="n">user</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span>
+ <span class="n">password</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span>
+ <span class="n">dbname</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="ow">or</span> <span class="n">conn</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span>
+ <span class="n">port</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">)</span>
+ <span class="c1"># check for ssl parameters in conn.extra</span>
+ <span class="k">for</span> <span class="n">arg_name</span><span class="p">,</span> <span class="n">arg_val</span> <span class="ow">in</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+ <span class="k">if</span> <span class="n">arg_name</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'sslmode'</span><span class="p">,</span> <span class="s1">'sslcert'</span><span class="p">,</span> <span class="s1">'sslkey'</span><span class="p">,</span>
+ <span class="s1">'sslrootcert'</span><span class="p">,</span> <span class="s1">'sslcrl'</span><span class="p">,</span> <span class="s1">'application_name'</span><span class="p">,</span>
+ <span class="s1">'keepalives_idle'</span><span class="p">]:</span>
+ <span class="n">conn_args</span><span class="p">[</span><span class="n">arg_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">arg_val</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="n">psycopg2</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="o">**</span><span class="n">conn_args</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span>
+
+<div class="viewcode-block" id="PostgresHook.copy_expert"><a class="viewcode-back" href="../../../code.html#airflow.hooks.postgres_hook.PostgresHook.copy_expert">[docs]</a> <span class="k">def</span> <span class="nf">copy_expert</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</span> <span class="nb">open</span><span class="o">=</span><span class="nb">open</span><span class="p">):</span>
+ <span class="sd">'''</span>
+<span class="sd"> Executes SQL using psycopg2 copy_expert method</span>
+<span class="sd"> Necessary to execute COPY command without access to a superuser</span>
+<span class="sd"> '''</span>
+ <span class="n">f</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s1">'w'</span><span class="p">)</span>
+ <span class="k">with</span> <span class="n">closing</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">())</span> <span class="k">as</span> <span class="n">conn</span><span class="p">:</span>
+ <span class="k">with</span> <span class="n">closing</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">())</span> <span class="k">as</span> <span class="n">cur</span><span class="p">:</span>
+ <span class="n">cur</span><span class="o">.</span><span class="n">copy_expert</span><span class="p">(</span><span class="n">sql</span><span class="p">,</span> <span class="n">f</span><span class="p">)</span></div>
+
+ <span class="nd">@staticmethod</span>
+ <span class="k">def</span> <span class="nf">_serialize_cell</span><span class="p">(</span><span class="n">cell</span><span class="p">,</span> <span class="n">conn</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Postgresql will adapt all arguments to the execute() method internally,</span>
+<span class="sd"> hence we return cell without any conversion.</span>
+
+<span class="sd"> See http://initd.org/psycopg/docs/advanced.html#adapting-new-types for</span>
+<span class="sd"> more information.</span>
+
+<span class="sd"> :param cell: The cell to insert into the table</span>
+<span class="sd"> :type cell: object</span>
+<span class="sd"> :param conn: The database connection</span>
+<span class="sd"> :type conn: connection object</span>
+<span class="sd"> :return: The cell</span>
+<span class="sd"> :rtype: object</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="n">cell</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/hooks/presto_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/hooks/presto_hook.html b/_modules/airflow/hooks/presto_hook.html
new file mode 100644
index 0000000..ab441fd
--- /dev/null
+++ b/_modules/airflow/hooks/presto_hook.html
@@ -0,0 +1,366 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.hooks.presto_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.hooks.presto_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.hooks.presto_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
+
+<span class="kn">from</span> <span class="nn">pyhive</span> <span class="k">import</span> <span class="n">presto</span>
+<span class="kn">from</span> <span class="nn">pyhive.exc</span> <span class="k">import</span> <span class="n">DatabaseError</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="k">import</span> <span class="n">DbApiHook</span>
+
+<span class="k">class</span> <span class="nc">PrestoException</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
+ <span class="k">pass</span>
+
+
+<div class="viewcode-block" id="PrestoHook"><a class="viewcode-back" href="../../../code.html#airflow.hooks.presto_hook.PrestoHook">[docs]</a><span class="k">class</span> <span class="nc">PrestoHook</span><span class="p">(</span><span class="n">DbApiHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with Presto through PyHive!</span>
+
+<span class="sd"> >>> ph = PrestoHook()</span>
+<span class="sd"> >>> sql = "SELECT count(1) AS num FROM airflow.static_babynames"</span>
+<span class="sd"> >>> ph.get_records(sql)</span>
+<span class="sd"> [[340698]]</span>
+<span class="sd"> """</span>
+
+ <span class="n">conn_name_attr</span> <span class="o">=</span> <span class="s1">'presto_conn_id'</span>
+ <span class="n">default_conn_name</span> <span class="o">=</span> <span class="s1">'presto_default'</span>
+
+<div class="viewcode-block" id="PrestoHook.get_conn"><a class="viewcode-back" href="../../../code.html#airflow.hooks.presto_hook.PrestoHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""Returns a connection object"""</span>
+ <span class="n">db</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">presto_conn_id</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">presto</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span>
+ <span class="n">host</span><span class="o">=</span><span class="n">db</span><span class="o">.</span><span class="n">host</span><span class="p">,</span>
+ <span class="n">port</span><span class="o">=</span><span class="n">db</span><span class="o">.</span><span class="n">port</span><span class="p">,</span>
+ <span class="n">username</span><span class="o">=</span><span class="n">db</span><span class="o">.</span><span class="n">login</span><span class="p">,</span>
+ <span class="n">catalog</span><span class="o">=</span><span class="n">db</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'catalog'</span><span class="p">,</span> <span class="s1">'hive'</span><span class="p">),</span>
+ <span class="n">schema</span><span class="o">=</span><span class="n">db</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span></div>
+
+ <span class="nd">@staticmethod</span>
+ <span class="k">def</span> <span class="nf">_strip_sql</span><span class="p">(</span><span class="n">sql</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">sql</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s1">';'</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_get_pretty_exception_message</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Parses some DatabaseError to provide a better error message</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="p">(</span><span class="nb">hasattr</span><span class="p">(</span><span class="n">e</span><span class="p">,</span> <span class="s1">'message'</span><span class="p">)</span>
+ <span class="ow">and</span> <span class="s1">'errorName'</span> <span class="ow">in</span> <span class="n">e</span><span class="o">.</span><span class="n">message</span>
+ <span class="ow">and</span> <span class="s1">'message'</span> <span class="ow">in</span> <span class="n">e</span><span class="o">.</span><span class="n">message</span><span class="p">):</span>
+ <span class="k">return</span> <span class="p">(</span><span class="s1">'</span><span class="si">{name}</span><span class="s1">: </span><span class="si">{message}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">name</span><span class="o">=</span><span class="n">e</span><span class="o">.</span><span class="n">message</span><span class="p">[</span><span class="s1">'errorName'</span><span class="p">],</span>
+ <span class="n">message</span><span class="o">=</span><span class="n">e</span><span class="o">.</span><span class="n">message</span><span class="p">[</span><span class="s1">'message'</span><span class="p">]))</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
+
+<div class="viewcode-block" id="PrestoHook.get_records"><a class="viewcode-back" href="../../../code.html#airflow.hooks.presto_hook.PrestoHook.get_records">[docs]</a> <span class="k">def</span> <span class="nf">get_records</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Get a set of records from Presto</span>
+<span class="sd"> """</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">PrestoHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">get_records</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_strip_sql</span><span class="p">(</span><span class="n">hql</span><span class="p">),</span> <span class="n">parameters</span><span class="p">)</span>
+ <span class="k">except</span> <span class="n">DatabaseError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_get_pretty_exception_message</span><span class="p">(</span><span class="n">e</span><span class="p">))</span></div>
+
+<div class="viewcode-block" id="PrestoHook.get_first"><a class="viewcode-back" href="../../../code.html#airflow.hooks.presto_hook.PrestoHook.get_first">[docs]</a> <span class="k">def</span> <span class="nf">get_first</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns only the first row, regardless of how many rows the query</span>
+<span class="sd"> returns.</span>
+<span class="sd"> """</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">PrestoHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">get_first</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_strip_sql</span><span class="p">(</span><span class="n">hql</span><span class="p">),</span> <span class="n">parameters</span><span class="p">)</span>
+ <span class="k">except</span> <span class="n">DatabaseError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_get_pretty_exception_message</span><span class="p">(</span><span class="n">e</span><span class="p">))</span></div>
+
+<div class="viewcode-block" id="PrestoHook.get_pandas_df"><a class="viewcode-back" href="../../../code.html#airflow.hooks.presto_hook.PrestoHook.get_pandas_df">[docs]</a> <span class="k">def</span> <span class="nf">get_pandas_df</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Get a pandas dataframe from a sql query.</span>
+<span class="sd"> """</span>
+ <span class="kn">import</span> <span class="nn">pandas</span>
+ <span class="n">cursor</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_cursor</span><span class="p">()</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">cursor</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_strip_sql</span><span class="p">(</span><span class="n">hql</span><span class="p">),</span> <span class="n">parameters</span><span class="p">)</span>
+ <span class="n">data</span> <span class="o">=</span> <span class="n">cursor</span><span class="o">.</span><span class="n">fetchall</span><span class="p">()</span>
+ <span class="k">except</span> <span class="n">DatabaseError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_get_pretty_exception_message</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
+ <span class="n">column_descriptions</span> <span class="o">=</span> <span class="n">cursor</span><span class="o">.</span><span class="n">description</span>
+ <span class="k">if</span> <span class="n">data</span><span class="p">:</span>
+ <span class="n">df</span> <span class="o">=</span> <span class="n">pandas</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
+ <span class="n">df</span><span class="o">.</span><span class="n">columns</span> <span class="o">=</span> <span class="p">[</span><span class="n">c</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">column_descriptions</span><span class="p">]</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">df</span> <span class="o">=</span> <span class="n">pandas</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">df</span></div>
+
+<div class="viewcode-block" id="PrestoHook.run"><a class="viewcode-back" href="../../../code.html#airflow.hooks.presto_hook.PrestoHook.run">[docs]</a> <span class="k">def</span> <span class="nf">run</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Execute the statement against Presto. Can be used to create views.</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">PrestoHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_strip_sql</span><span class="p">(</span><span class="n">hql</span><span class="p">),</span> <span class="n">parameters</span><span class="p">)</span></div>
+
+ <span class="c1"># TODO Enable commit_every once PyHive supports transaction.</span>
+ <span class="c1"># Unfortunately, PyHive 0.5.1 doesn't support transaction for now,</span>
+ <span class="c1"># whereas Presto 0.132+ does.</span>
+<div class="viewcode-block" id="PrestoHook.insert_rows"><a class="viewcode-back" href="../../../code.html#airflow.hooks.presto_hook.PrestoHook.insert_rows">[docs]</a> <span class="k">def</span> <span class="nf">insert_rows</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">rows</span><span class="p">,</span> <span class="n">target_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> A generic way to insert a set of tuples into a table.</span>
+
+<span class="sd"> :param table: Name of the target table</span>
+<span class="sd"> :type table: str</span>
+<span class="sd"> :param rows: The rows to insert into the table</span>
+<span class="sd"> :type rows: iterable of tuples</span>
+<span class="sd"> :param target_fields: The names of the columns to fill in the table</span>
+<span class="sd"> :type target_fields: iterable of strings</span>
+<span class="sd"> """</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">PrestoHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">insert_rows</span><span class="p">(</span><span class="n">table</span><span class="p">,</span> <span class="n">rows</span><span class="p">,</span> <span class="n">target_fields</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[30/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/gcs_to_bq.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/gcs_to_bq.html b/_modules/airflow/contrib/operators/gcs_to_bq.html
index 1d61268..db2b756 100644
--- a/_modules/airflow/contrib/operators/gcs_to_bq.html
+++ b/_modules/airflow/contrib/operators/gcs_to_bq.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,22 @@
<h1>Source code for airflow.contrib.operators.gcs_to_bq</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="kn">import</span> <span class="nn">json</span>
@@ -192,6 +199,92 @@
<div class="viewcode-block" id="GoogleCloudStorageToBigQueryOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcs_to_bq.GoogleCloudStorageToBigQueryOperator">[docs]</a><span class="k">class</span> <span class="nc">GoogleCloudStorageToBigQueryOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Loads files from Google cloud storage into BigQuery.</span>
+
+<span class="sd"> The schema to be used for the BigQuery table may be specified in one of</span>
+<span class="sd"> two ways. You may either directly pass the schema fields in, or you may</span>
+<span class="sd"> point the operator to a Google cloud storage object name. The object in</span>
+<span class="sd"> Google cloud storage must be a JSON file with the schema fields in it.</span>
+
+<span class="sd"> :param bucket: The bucket to load from.</span>
+<span class="sd"> :type bucket: string</span>
+<span class="sd"> :param source_objects: List of Google cloud storage URIs to load from.</span>
+<span class="sd"> If source_format is 'DATASTORE_BACKUP', the list must only contain a single URI.</span>
+<span class="sd"> :type object: list</span>
+<span class="sd"> :param destination_project_dataset_table: The dotted (<project>.)<dataset>.<table></span>
+<span class="sd"> BigQuery table to load data into. If <project> is not included, project will</span>
+<span class="sd"> be the project defined in the connection json.</span>
+<span class="sd"> :type destination_project_dataset_table: string</span>
+<span class="sd"> :param schema_fields: If set, the schema field list as defined here:</span>
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load</span>
+<span class="sd"> Should not be set when source_format is 'DATASTORE_BACKUP'.</span>
+<span class="sd"> :type schema_fields: list</span>
+<span class="sd"> :param schema_object: If set, a GCS object path pointing to a .json file that</span>
+<span class="sd"> contains the schema for the table.</span>
+<span class="sd"> :param schema_object: string</span>
+<span class="sd"> :param source_format: File format to export.</span>
+<span class="sd"> :type source_format: string</span>
+<span class="sd"> :param compression: [Optional] The compression type of the data source.</span>
+<span class="sd"> Possible values include GZIP and NONE.</span>
+<span class="sd"> The default value is NONE.</span>
+<span class="sd"> This setting is ignored for Google Cloud Bigtable,</span>
+<span class="sd"> Google Cloud Datastore backups and Avro formats.</span>
+<span class="sd"> :type compression: string</span>
+<span class="sd"> :param create_disposition: The create disposition if the table doesn't exist.</span>
+<span class="sd"> :type create_disposition: string</span>
+<span class="sd"> :param skip_leading_rows: Number of rows to skip when loading from a CSV.</span>
+<span class="sd"> :type skip_leading_rows: int</span>
+<span class="sd"> :param write_disposition: The write disposition if the table already exists.</span>
+<span class="sd"> :type write_disposition: string</span>
+<span class="sd"> :param field_delimiter: The delimiter to use when loading from a CSV.</span>
+<span class="sd"> :type field_delimiter: string</span>
+<span class="sd"> :param max_bad_records: The maximum number of bad records that BigQuery can</span>
+<span class="sd"> ignore when running the job.</span>
+<span class="sd"> :type max_bad_records: int</span>
+<span class="sd"> :param quote_character: The value that is used to quote data sections in a CSV file.</span>
+<span class="sd"> :type quote_character: string</span>
+<span class="sd"> :param ignore_unknown_values: [Optional] Indicates if BigQuery should allow</span>
+<span class="sd"> extra values that are not represented in the table schema.</span>
+<span class="sd"> If true, the extra values are ignored. If false, records with extra columns</span>
+<span class="sd"> are treated as bad records, and if there are too many bad records, an</span>
+<span class="sd"> invalid error is returned in the job result.</span>
+<span class="sd"> :type ignore_unknown_values: bool</span>
+<span class="sd"> :param allow_quoted_newlines: Whether to allow quoted newlines (true) or not (false).</span>
+<span class="sd"> :type allow_quoted_newlines: boolean</span>
+<span class="sd"> :param allow_jagged_rows: Accept rows that are missing trailing optional columns.</span>
+<span class="sd"> The missing values are treated as nulls. If false, records with missing trailing</span>
+<span class="sd"> columns are treated as bad records, and if there are too many bad records, an</span>
+<span class="sd"> invalid error is returned in the job result. Only applicable to CSV, ignored</span>
+<span class="sd"> for other formats.</span>
+<span class="sd"> :type allow_jagged_rows: bool</span>
+<span class="sd"> :param max_id_key: If set, the name of a column in the BigQuery table</span>
+<span class="sd"> that's to be loaded. Thsi will be used to select the MAX value from</span>
+<span class="sd"> BigQuery after the load occurs. The results will be returned by the</span>
+<span class="sd"> execute() command, which in turn gets stored in XCom for future</span>
+<span class="sd"> operators to use. This can be helpful with incremental loads--during</span>
+<span class="sd"> future executions, you can pick up from the max ID.</span>
+<span class="sd"> :type max_id_key: string</span>
+<span class="sd"> :param bigquery_conn_id: Reference to a specific BigQuery hook.</span>
+<span class="sd"> :type bigquery_conn_id: string</span>
+<span class="sd"> :param google_cloud_storage_conn_id: Reference to a specific Google</span>
+<span class="sd"> cloud storage hook.</span>
+<span class="sd"> :type google_cloud_storage_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any. For this to</span>
+<span class="sd"> work, the service account making the request must have domain-wide</span>
+<span class="sd"> delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> :param schema_update_options: Allows the schema of the destination</span>
+<span class="sd"> table to be updated as a side effect of the load job.</span>
+<span class="sd"> :type schema_update_options: list</span>
+<span class="sd"> :param src_fmt_configs: configure optional fields specific to the source format</span>
+<span class="sd"> :type src_fmt_configs: dict</span>
+<span class="sd"> :param external_table: Flag to specify if the destination table should be</span>
+<span class="sd"> a BigQuery external table. Default Value is False.</span>
+<span class="sd"> :type external_table: bool</span>
+<span class="sd"> :param time_partitioning: configure optional time partitioning fields i.e.</span>
+<span class="sd"> partition by field, type and expiration as per API specifications.</span>
+<span class="sd"> Note that 'field' is not available in concurrency with</span>
+<span class="sd"> dataset.table$partition.</span>
+<span class="sd"> :type time_partitioning: dict</span>
<span class="sd"> """</span>
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bucket'</span><span class="p">,</span> <span class="s1">'source_objects'</span><span class="p">,</span>
<span class="s1">'schema_object'</span><span class="p">,</span> <span class="s1">'destination_project_dataset_table'</span><span class="p">)</span>
@@ -199,96 +292,33 @@
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f0eee4'</span>
<span class="nd">@apply_defaults</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
- <span class="bp">self</span><span class="p">,</span>
- <span class="n">bucket</span><span class="p">,</span>
- <span class="n">source_objects</span><span class="p">,</span>
- <span class="n">destination_project_dataset_table</span><span class="p">,</span>
- <span class="n">schema_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">schema_object</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">source_format</span><span class="o">=</span><span class="s1">'CSV'</span><span class="p">,</span>
- <span class="n">create_disposition</span><span class="o">=</span><span class="s1">'CREATE_IF_NEEDED'</span><span class="p">,</span>
- <span class="n">skip_leading_rows</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
- <span class="n">write_disposition</span><span class="o">=</span><span class="s1">'WRITE_EMPTY'</span><span class="p">,</span>
- <span class="n">field_delimiter</span><span class="o">=</span><span class="s1">','</span><span class="p">,</span>
- <span class="n">max_bad_records</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
- <span class="n">quote_character</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">allow_quoted_newlines</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">allow_jagged_rows</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">max_id_key</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">'bigquery_default'</span><span class="p">,</span>
- <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_storage_default'</span><span class="p">,</span>
- <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">schema_update_options</span><span class="o">=</span><span class="p">(),</span>
- <span class="n">src_fmt_configs</span><span class="o">=</span><span class="p">{},</span>
- <span class="o">*</span><span class="n">args</span><span class="p">,</span>
- <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> The schema to be used for the BigQuery table may be specified in one of</span>
-<span class="sd"> two ways. You may either directly pass the schema fields in, or you may</span>
-<span class="sd"> point the operator to a Google cloud storage object name. The object in</span>
-<span class="sd"> Google cloud storage must be a JSON file with the schema fields in it.</span>
-
-<span class="sd"> :param bucket: The bucket to load from.</span>
-<span class="sd"> :type bucket: string</span>
-<span class="sd"> :param source_objects: List of Google cloud storage URIs to load from.</span>
-<span class="sd"> If source_format is 'DATASTORE_BACKUP', the list must only contain a single URI.</span>
-<span class="sd"> :type object: list</span>
-<span class="sd"> :param destination_project_dataset_table: The dotted (<project>.)<dataset>.<table></span>
-<span class="sd"> BigQuery table to load data into. If <project> is not included, project will</span>
-<span class="sd"> be the project defined in the connection json.</span>
-<span class="sd"> :type destination_project_dataset_table: string</span>
-<span class="sd"> :param schema_fields: If set, the schema field list as defined here:</span>
-<span class="sd"> https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load</span>
-<span class="sd"> Should not be set when source_format is 'DATASTORE_BACKUP'.</span>
-<span class="sd"> :type schema_fields: list</span>
-<span class="sd"> :param schema_object: If set, a GCS object path pointing to a .json file that</span>
-<span class="sd"> contains the schema for the table.</span>
-<span class="sd"> :param schema_object: string</span>
-<span class="sd"> :param source_format: File format to export.</span>
-<span class="sd"> :type source_format: string</span>
-<span class="sd"> :param create_disposition: The create disposition if the table doesn't exist.</span>
-<span class="sd"> :type create_disposition: string</span>
-<span class="sd"> :param skip_leading_rows: Number of rows to skip when loading from a CSV.</span>
-<span class="sd"> :type skip_leading_rows: int</span>
-<span class="sd"> :param write_disposition: The write disposition if the table already exists.</span>
-<span class="sd"> :type write_disposition: string</span>
-<span class="sd"> :param field_delimiter: The delimiter to use when loading from a CSV.</span>
-<span class="sd"> :type field_delimiter: string</span>
-<span class="sd"> :param max_bad_records: The maximum number of bad records that BigQuery can</span>
-<span class="sd"> ignore when running the job.</span>
-<span class="sd"> :type max_bad_records: int</span>
-<span class="sd"> :param quote_character: The value that is used to quote data sections in a CSV file.</span>
-<span class="sd"> :type quote_character: string</span>
-<span class="sd"> :param allow_quoted_newlines: Whether to allow quoted newlines (true) or not (false).</span>
-<span class="sd"> :type allow_quoted_newlines: boolean</span>
-<span class="sd"> :param allow_jagged_rows: Accept rows that are missing trailing optional columns.</span>
-<span class="sd"> The missing values are treated as nulls. If false, records with missing trailing columns</span>
-<span class="sd"> are treated as bad records, and if there are too many bad records, an invalid error is</span>
-<span class="sd"> returned in the job result. Only applicable to CSV, ignored for other formats.</span>
-<span class="sd"> :type allow_jagged_rows: bool</span>
-<span class="sd"> :param max_id_key: If set, the name of a column in the BigQuery table</span>
-<span class="sd"> that's to be loaded. Thsi will be used to select the MAX value from</span>
-<span class="sd"> BigQuery after the load occurs. The results will be returned by the</span>
-<span class="sd"> execute() command, which in turn gets stored in XCom for future</span>
-<span class="sd"> operators to use. This can be helpful with incremental loads--during</span>
-<span class="sd"> future executions, you can pick up from the max ID.</span>
-<span class="sd"> :type max_id_key: string</span>
-<span class="sd"> :param bigquery_conn_id: Reference to a specific BigQuery hook.</span>
-<span class="sd"> :type bigquery_conn_id: string</span>
-<span class="sd"> :param google_cloud_storage_conn_id: Reference to a specific Google</span>
-<span class="sd"> cloud storage hook.</span>
-<span class="sd"> :type google_cloud_storage_conn_id: string</span>
-<span class="sd"> :param delegate_to: The account to impersonate, if any. For this to</span>
-<span class="sd"> work, the service account making the request must have domain-wide</span>
-<span class="sd"> delegation enabled.</span>
-<span class="sd"> :type delegate_to: string</span>
-<span class="sd"> :param schema_update_options: Allows the schema of the desitination</span>
-<span class="sd"> table to be updated as a side effect of the load job.</span>
-<span class="sd"> :type schema_update_options: list</span>
-<span class="sd"> :param src_fmt_configs: configure optional fields specific to the source format</span>
-<span class="sd"> :type src_fmt_configs: dict</span>
-<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">bucket</span><span class="p">,</span>
+ <span class="n">source_objects</span><span class="p">,</span>
+ <span class="n">destination_project_dataset_table</span><span class="p">,</span>
+ <span class="n">schema_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">schema_object</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">source_format</span><span class="o">=</span><span class="s1">'CSV'</span><span class="p">,</span>
+ <span class="n">compression</span><span class="o">=</span><span class="s1">'NONE'</span><span class="p">,</span>
+ <span class="n">create_disposition</span><span class="o">=</span><span class="s1">'CREATE_IF_NEEDED'</span><span class="p">,</span>
+ <span class="n">skip_leading_rows</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
+ <span class="n">write_disposition</span><span class="o">=</span><span class="s1">'WRITE_EMPTY'</span><span class="p">,</span>
+ <span class="n">field_delimiter</span><span class="o">=</span><span class="s1">','</span><span class="p">,</span>
+ <span class="n">max_bad_records</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
+ <span class="n">quote_character</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">ignore_unknown_values</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">allow_quoted_newlines</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">allow_jagged_rows</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">max_id_key</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">'bigquery_default'</span><span class="p">,</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">schema_update_options</span><span class="o">=</span><span class="p">(),</span>
+ <span class="n">src_fmt_configs</span><span class="o">=</span><span class="p">{},</span>
+ <span class="n">external_table</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">time_partitioning</span><span class="o">=</span><span class="p">{},</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+
<span class="nb">super</span><span class="p">(</span><span class="n">GoogleCloudStorageToBigQueryOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="c1"># GCS config</span>
@@ -300,14 +330,17 @@
<span class="bp">self</span><span class="o">.</span><span class="n">destination_project_dataset_table</span> <span class="o">=</span> <span class="n">destination_project_dataset_table</span>
<span class="bp">self</span><span class="o">.</span><span class="n">schema_fields</span> <span class="o">=</span> <span class="n">schema_fields</span>
<span class="bp">self</span><span class="o">.</span><span class="n">source_format</span> <span class="o">=</span> <span class="n">source_format</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">compression</span> <span class="o">=</span> <span class="n">compression</span>
<span class="bp">self</span><span class="o">.</span><span class="n">create_disposition</span> <span class="o">=</span> <span class="n">create_disposition</span>
<span class="bp">self</span><span class="o">.</span><span class="n">skip_leading_rows</span> <span class="o">=</span> <span class="n">skip_leading_rows</span>
<span class="bp">self</span><span class="o">.</span><span class="n">write_disposition</span> <span class="o">=</span> <span class="n">write_disposition</span>
<span class="bp">self</span><span class="o">.</span><span class="n">field_delimiter</span> <span class="o">=</span> <span class="n">field_delimiter</span>
<span class="bp">self</span><span class="o">.</span><span class="n">max_bad_records</span> <span class="o">=</span> <span class="n">max_bad_records</span>
<span class="bp">self</span><span class="o">.</span><span class="n">quote_character</span> <span class="o">=</span> <span class="n">quote_character</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">ignore_unknown_values</span> <span class="o">=</span> <span class="n">ignore_unknown_values</span>
<span class="bp">self</span><span class="o">.</span><span class="n">allow_quoted_newlines</span> <span class="o">=</span> <span class="n">allow_quoted_newlines</span>
<span class="bp">self</span><span class="o">.</span><span class="n">allow_jagged_rows</span> <span class="o">=</span> <span class="n">allow_jagged_rows</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">external_table</span> <span class="o">=</span> <span class="n">external_table</span>
<span class="bp">self</span><span class="o">.</span><span class="n">max_id_key</span> <span class="o">=</span> <span class="n">max_id_key</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
@@ -316,6 +349,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">schema_update_options</span> <span class="o">=</span> <span class="n">schema_update_options</span>
<span class="bp">self</span><span class="o">.</span><span class="n">src_fmt_configs</span> <span class="o">=</span> <span class="n">src_fmt_configs</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">time_partitioning</span> <span class="o">=</span> <span class="n">time_partitioning</span>
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">bq_hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
@@ -336,21 +370,41 @@
<span class="k">for</span> <span class="n">source_object</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">source_objects</span><span class="p">]</span>
<span class="n">conn</span> <span class="o">=</span> <span class="n">bq_hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
<span class="n">cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
- <span class="n">cursor</span><span class="o">.</span><span class="n">run_load</span><span class="p">(</span>
- <span class="n">destination_project_dataset_table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">destination_project_dataset_table</span><span class="p">,</span>
- <span class="n">schema_fields</span><span class="o">=</span><span class="n">schema_fields</span><span class="p">,</span>
- <span class="n">source_uris</span><span class="o">=</span><span class="n">source_uris</span><span class="p">,</span>
- <span class="n">source_format</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">source_format</span><span class="p">,</span>
- <span class="n">create_disposition</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">create_disposition</span><span class="p">,</span>
- <span class="n">skip_leading_rows</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">skip_leading_rows</span><span class="p">,</span>
- <span class="n">write_disposition</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">write_disposition</span><span class="p">,</span>
- <span class="n">field_delimiter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">field_delimiter</span><span class="p">,</span>
- <span class="n">max_bad_records</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">max_bad_records</span><span class="p">,</span>
- <span class="n">quote_character</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">quote_character</span><span class="p">,</span>
- <span class="n">allow_quoted_newlines</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">allow_quoted_newlines</span><span class="p">,</span>
- <span class="n">allow_jagged_rows</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">allow_jagged_rows</span><span class="p">,</span>
- <span class="n">schema_update_options</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">schema_update_options</span><span class="p">,</span>
- <span class="n">src_fmt_configs</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">src_fmt_configs</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">external_table</span><span class="p">:</span>
+ <span class="n">cursor</span><span class="o">.</span><span class="n">create_external_table</span><span class="p">(</span>
+ <span class="n">external_project_dataset_table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">destination_project_dataset_table</span><span class="p">,</span>
+ <span class="n">schema_fields</span><span class="o">=</span><span class="n">schema_fields</span><span class="p">,</span>
+ <span class="n">source_uris</span><span class="o">=</span><span class="n">source_uris</span><span class="p">,</span>
+ <span class="n">source_format</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">source_format</span><span class="p">,</span>
+ <span class="n">compression</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">compression</span><span class="p">,</span>
+ <span class="n">skip_leading_rows</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">skip_leading_rows</span><span class="p">,</span>
+ <span class="n">field_delimiter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">field_delimiter</span><span class="p">,</span>
+ <span class="n">max_bad_records</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">max_bad_records</span><span class="p">,</span>
+ <span class="n">quote_character</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">quote_character</span><span class="p">,</span>
+ <span class="n">ignore_unknown_values</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">ignore_unknown_values</span><span class="p">,</span>
+ <span class="n">allow_quoted_newlines</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">allow_quoted_newlines</span><span class="p">,</span>
+ <span class="n">allow_jagged_rows</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">allow_jagged_rows</span><span class="p">,</span>
+ <span class="n">src_fmt_configs</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">src_fmt_configs</span>
+ <span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">cursor</span><span class="o">.</span><span class="n">run_load</span><span class="p">(</span>
+ <span class="n">destination_project_dataset_table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">destination_project_dataset_table</span><span class="p">,</span>
+ <span class="n">schema_fields</span><span class="o">=</span><span class="n">schema_fields</span><span class="p">,</span>
+ <span class="n">source_uris</span><span class="o">=</span><span class="n">source_uris</span><span class="p">,</span>
+ <span class="n">source_format</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">source_format</span><span class="p">,</span>
+ <span class="n">create_disposition</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">create_disposition</span><span class="p">,</span>
+ <span class="n">skip_leading_rows</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">skip_leading_rows</span><span class="p">,</span>
+ <span class="n">write_disposition</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">write_disposition</span><span class="p">,</span>
+ <span class="n">field_delimiter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">field_delimiter</span><span class="p">,</span>
+ <span class="n">max_bad_records</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">max_bad_records</span><span class="p">,</span>
+ <span class="n">quote_character</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">quote_character</span><span class="p">,</span>
+ <span class="n">ignore_unknown_values</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">ignore_unknown_values</span><span class="p">,</span>
+ <span class="n">allow_quoted_newlines</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">allow_quoted_newlines</span><span class="p">,</span>
+ <span class="n">allow_jagged_rows</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">allow_jagged_rows</span><span class="p">,</span>
+ <span class="n">schema_update_options</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">schema_update_options</span><span class="p">,</span>
+ <span class="n">src_fmt_configs</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">src_fmt_configs</span><span class="p">,</span>
+ <span class="n">time_partitioning</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">time_partitioning</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_id_key</span><span class="p">:</span>
<span class="n">cursor</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="s1">'SELECT MAX(</span><span class="si">{}</span><span class="s1">) FROM </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/gcs_to_gcs.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/gcs_to_gcs.html b/_modules/airflow/contrib/operators/gcs_to_gcs.html
new file mode 100644
index 0000000..7379d8a
--- /dev/null
+++ b/_modules/airflow/contrib/operators/gcs_to_gcs.html
@@ -0,0 +1,365 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.gcs_to_gcs — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.gcs_to_gcs</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.gcs_to_gcs</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcs_hook</span> <span class="k">import</span> <span class="n">GoogleCloudStorageHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="GoogleCloudStorageToGoogleCloudStorageOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcs_to_gcs.GoogleCloudStorageToGoogleCloudStorageOperator">[docs]</a><span class="k">class</span> <span class="nc">GoogleCloudStorageToGoogleCloudStorageOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Copies objects from a bucket to another, with renaming if requested.</span>
+
+<span class="sd"> :param source_bucket: The source Google cloud storage bucket where the object is.</span>
+<span class="sd"> :type source_bucket: string</span>
+<span class="sd"> :param source_object: The source name of the object to copy in the Google cloud</span>
+<span class="sd"> storage bucket.</span>
+<span class="sd"> If wildcards are used in this argument:</span>
+<span class="sd"> You can use only one wildcard for objects (filenames) within your</span>
+<span class="sd"> bucket. The wildcard can appear inside the object name or at the</span>
+<span class="sd"> end of the object name. Appending a wildcard to the bucket name is</span>
+<span class="sd"> unsupported.</span>
+<span class="sd"> :type source_object: string</span>
+<span class="sd"> :param destination_bucket: The destination Google cloud storage bucket</span>
+<span class="sd"> where the object should be.</span>
+<span class="sd"> :type destination_bucket: string</span>
+<span class="sd"> :param destination_object: The destination name of the object in the</span>
+<span class="sd"> destination Google cloud</span>
+<span class="sd"> storage bucket.</span>
+<span class="sd"> If a wildcard is supplied in the source_object argument, this is the</span>
+<span class="sd"> prefix that will be prepended to the final destination objects' paths.</span>
+<span class="sd"> :type destination_object: string</span>
+<span class="sd"> :param move_object: When move object is True, the object is moved instead</span>
+<span class="sd"> of copied to the new location.</span>
+<span class="sd"> This is the equivalent of a mv command as opposed to a</span>
+<span class="sd"> cp command.</span>
+<span class="sd"> :type move_object: bool</span>
+<span class="sd"> :param google_cloud_storage_conn_id: The connection ID to use when</span>
+<span class="sd"> connecting to Google cloud storage.</span>
+<span class="sd"> :type google_cloud_storage_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request must have</span>
+<span class="sd"> domain-wide delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'source_bucket'</span><span class="p">,</span> <span class="s1">'source_object'</span><span class="p">,</span> <span class="s1">'destination_bucket'</span><span class="p">,</span>
+ <span class="s1">'destination_object'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f0eee4'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">source_bucket</span><span class="p">,</span>
+ <span class="n">source_object</span><span class="p">,</span>
+ <span class="n">destination_bucket</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">destination_object</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">move_object</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">GoogleCloudStorageToGoogleCloudStorageOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">source_bucket</span> <span class="o">=</span> <span class="n">source_bucket</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">source_object</span> <span class="o">=</span> <span class="n">source_object</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">destination_bucket</span> <span class="o">=</span> <span class="n">destination_bucket</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">destination_object</span> <span class="o">=</span> <span class="n">destination_object</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">move_object</span> <span class="o">=</span> <span class="n">move_object</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span> <span class="o">=</span> <span class="n">google_cloud_storage_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span>
+ <span class="p">)</span>
+
+ <span class="k">if</span> <span class="s1">'*'</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">source_object</span><span class="p">:</span>
+ <span class="n">wildcard_position</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">source_object</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="s1">'*'</span><span class="p">)</span>
+ <span class="n">objects</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">source_bucket</span><span class="p">,</span>
+ <span class="n">prefix</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">source_object</span><span class="p">[:</span><span class="n">wildcard_position</span><span class="p">],</span>
+ <span class="n">delimiter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">source_object</span><span class="p">[</span><span class="n">wildcard_position</span> <span class="o">+</span> <span class="mi">1</span><span class="p">:])</span>
+
+ <span class="k">for</span> <span class="n">source_object</span> <span class="ow">in</span> <span class="n">objects</span><span class="p">:</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">destination_object</span><span class="p">:</span>
+ <span class="n">destination_object</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{}</span><span class="s2">/</span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">destination_object</span><span class="p">,</span>
+ <span class="n">source_object</span><span class="p">[</span><span class="n">wildcard_position</span><span class="p">:])</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">destination_object</span> <span class="o">=</span> <span class="n">source_object</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing copy of gs://</span><span class="si">{0}</span><span class="s1">/</span><span class="si">{1}</span><span class="s1"> to '</span>
+ <span class="s1">'gs://</span><span class="si">{2}</span><span class="s1">/</span><span class="si">{3}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">source_bucket</span><span class="p">,</span>
+ <span class="n">source_object</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">destination_bucket</span><span class="p">,</span>
+ <span class="n">destination_object</span><span class="p">))</span>
+
+ <span class="n">hook</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">source_bucket</span><span class="p">,</span> <span class="n">source_object</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">destination_bucket</span><span class="p">,</span> <span class="n">destination_object</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">move_object</span><span class="p">:</span>
+ <span class="n">hook</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">source_bucket</span><span class="p">,</span> <span class="n">source_object</span><span class="p">)</span>
+
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'Executing copy of gs://</span><span class="si">{0}</span><span class="s1">/</span><span class="si">{1}</span><span class="s1"> to '</span>
+ <span class="s1">'gs://</span><span class="si">{2}</span><span class="s1">/</span><span class="si">{3}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">source_bucket</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">source_object</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">destination_bucket</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">source_bucket</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">destination_object</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">source_object</span>
+ <span class="p">)</span>
+ <span class="p">)</span>
+ <span class="n">hook</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">source_bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">source_object</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">destination_bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">destination_object</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">move_object</span><span class="p">:</span>
+ <span class="n">hook</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">source_bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">source_object</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/gcs_to_s3.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/gcs_to_s3.html b/_modules/airflow/contrib/operators/gcs_to_s3.html
new file mode 100644
index 0000000..14a6549
--- /dev/null
+++ b/_modules/airflow/contrib/operators/gcs_to_s3.html
@@ -0,0 +1,347 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.gcs_to_s3 — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.gcs_to_s3</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.gcs_to_s3</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcs_hook</span> <span class="k">import</span> <span class="n">GoogleCloudStorageHook</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.operators.gcs_list_operator</span> <span class="k">import</span> <span class="n">GoogleCloudStorageListOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.S3_hook</span> <span class="k">import</span> <span class="n">S3Hook</span>
+
+
+<div class="viewcode-block" id="GoogleCloudStorageToS3Operator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.gcs_to_s3.GoogleCloudStorageToS3Operator">[docs]</a><span class="k">class</span> <span class="nc">GoogleCloudStorageToS3Operator</span><span class="p">(</span><span class="n">GoogleCloudStorageListOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Synchronizes a Google Cloud Storage bucket with an S3 bucket.</span>
+
+<span class="sd"> :param bucket: The Google Cloud Storage bucket to find the objects.</span>
+<span class="sd"> :type bucket: string</span>
+<span class="sd"> :param prefix: Prefix string which filters objects whose name begin with this prefix</span>
+<span class="sd"> :type prefix: string</span>
+<span class="sd"> :param delimiter: The delimiter by which you want to filter the objects.</span>
+<span class="sd"> For e.g to lists the CSV files from in a directory in GCS you would use</span>
+<span class="sd"> delimiter='.csv'.</span>
+<span class="sd"> :type delimiter: string</span>
+<span class="sd"> :param google_cloud_storage_conn_id: The connection ID to use when</span>
+<span class="sd"> connecting to Google Cloud Storage.</span>
+<span class="sd"> :type google_cloud_storage_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request must have</span>
+<span class="sd"> domain-wide delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> :param dest_aws_conn_id: The destination S3 connection</span>
+<span class="sd"> :type dest_aws_conn_id: str</span>
+<span class="sd"> :param dest_s3_key: The base S3 key to be used to store the files</span>
+<span class="sd"> :type dest_s3_key: str</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bucket'</span><span class="p">,</span> <span class="s1">'prefix'</span><span class="p">,</span> <span class="s1">'delimiter'</span><span class="p">,</span> <span class="s1">'dest_s3_key'</span><span class="p">)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f0eee4'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">bucket</span><span class="p">,</span>
+ <span class="n">prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">delimiter</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_storage_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">dest_aws_conn_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">dest_s3_key</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">replace</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+
+ <span class="nb">super</span><span class="p">(</span><span class="n">GoogleCloudStorageToS3Operator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+ <span class="n">bucket</span><span class="o">=</span><span class="n">bucket</span><span class="p">,</span>
+ <span class="n">prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">,</span>
+ <span class="n">delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">,</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="n">delegate_to</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span>
+ <span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">dest_aws_conn_id</span> <span class="o">=</span> <span class="n">dest_aws_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">dest_s3_key</span> <span class="o">=</span> <span class="n">dest_s3_key</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">replace</span> <span class="o">=</span> <span class="n">replace</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="c1"># use the super to list all files in an Google Cloud Storage bucket</span>
+ <span class="n">files</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">GoogleCloudStorageToS3Operator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">context</span><span class="p">)</span>
+ <span class="n">s3_hook</span> <span class="o">=</span> <span class="n">S3Hook</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dest_aws_conn_id</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">replace</span><span class="p">:</span>
+ <span class="c1"># if we are not replacing -> list all files in the S3 bucket</span>
+ <span class="c1"># and only keep those files which are present in</span>
+ <span class="c1"># Google Cloud Storage and not in S3</span>
+ <span class="n">bucket_name</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">S3Hook</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dest_s3_key</span><span class="p">)</span>
+ <span class="n">existing_files</span> <span class="o">=</span> <span class="n">s3_hook</span><span class="o">.</span><span class="n">list_keys</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">)</span>
+ <span class="n">files</span> <span class="o">=</span> <span class="nb">set</span><span class="p">(</span><span class="n">files</span><span class="p">)</span> <span class="o">-</span> <span class="nb">set</span><span class="p">(</span><span class="n">existing_files</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">files</span><span class="p">:</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span>
+ <span class="p">)</span>
+
+ <span class="k">for</span> <span class="n">file</span> <span class="ow">in</span> <span class="n">files</span><span class="p">:</span>
+ <span class="n">file_bytes</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">download</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="n">file</span><span class="p">)</span>
+
+ <span class="n">dest_key</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dest_s3_key</span> <span class="o">+</span> <span class="n">file</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Saving file to </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">dest_key</span><span class="p">)</span>
+
+ <span class="n">s3_hook</span><span class="o">.</span><span class="n">load_bytes</span><span class="p">(</span><span class="n">file_bytes</span><span class="p">,</span>
+ <span class="n">key</span><span class="o">=</span><span class="n">dest_key</span><span class="p">,</span>
+ <span class="n">replace</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">replace</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"All done, uploaded </span><span class="si">%d</span><span class="s2"> files to S3"</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">files</span><span class="p">))</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"In sync, no files needed to be uploaded to S3"</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">files</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/hipchat_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/hipchat_operator.html b/_modules/airflow/contrib/operators/hipchat_operator.html
index 1e04bf3..4f218b7 100644
--- a/_modules/airflow/contrib/operators/hipchat_operator.html
+++ b/_modules/airflow/contrib/operators/hipchat_operator.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,22 @@
<h1>Source code for airflow.contrib.operators.hipchat_operator</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
@@ -216,14 +223,14 @@
<span class="bp">self</span><span class="o">.</span><span class="n">url</span> <span class="o">=</span> <span class="kc">None</span>
<span class="bp">self</span><span class="o">.</span><span class="n">body</span> <span class="o">=</span> <span class="kc">None</span>
- <span class="k">def</span> <span class="nf">prepare_request</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<div class="viewcode-block" id="HipChatAPIOperator.prepare_request"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.hipchat_operator.HipChatAPIOperator.prepare_request">[docs]</a> <span class="k">def</span> <span class="nf">prepare_request</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Used by the execute function. Set the request method, url, and body of HipChat's</span>
<span class="sd"> REST API call.</span>
<span class="sd"> Override in child class. Each HipChatAPI child operator is responsible for having</span>
<span class="sd"> a prepare_request method call which sets self.method, self.url, and self.body.</span>
<span class="sd"> """</span>
- <span class="k">pass</span>
+ <span class="k">pass</span></div>
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">prepare_request</span><span class="p">()</span>
[20/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/sensors/hdfs_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/sensors/hdfs_sensor.html b/_modules/airflow/contrib/sensors/hdfs_sensor.html
new file mode 100644
index 0000000..df9ecaa
--- /dev/null
+++ b/_modules/airflow/contrib/sensors/hdfs_sensor.html
@@ -0,0 +1,313 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.sensors.hdfs_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.sensors.hdfs_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.sensors.hdfs_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">from</span> <span class="nn">airflow.sensors.hdfs_sensor</span> <span class="k">import</span> <span class="n">HdfsSensor</span>
+
+
+<div class="viewcode-block" id="HdfsSensorRegex"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.sensors.hdfs_sensor.HdfsSensorRegex">[docs]</a><span class="k">class</span> <span class="nc">HdfsSensorRegex</span><span class="p">(</span><span class="n">HdfsSensor</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">regex</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HdfsSensorRegex</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">regex</span> <span class="o">=</span> <span class="n">regex</span>
+
+<div class="viewcode-block" id="HdfsSensorRegex.poke"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.sensors.hdfs_sensor.HdfsSensorRegex.poke">[docs]</a> <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> poke matching files in a directory with self.regex</span>
+
+<span class="sd"> :return: Bool depending on the search criteria</span>
+<span class="sd"> """</span>
+ <span class="n">sb</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'Poking for </span><span class="si">{self.filepath}</span><span class="s1"> to be a directory '</span>
+ <span class="s1">'with files matching </span><span class="si">{self.regex.pattern}</span><span class="s1">'</span><span class="o">.</span>
+ <span class="nb">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
+ <span class="p">)</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="p">[</span><span class="n">f</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">sb</span><span class="o">.</span><span class="n">ls</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="p">],</span> <span class="n">include_toplevel</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span> <span class="k">if</span>
+ <span class="n">f</span><span class="p">[</span><span class="s1">'file_type'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'f'</span> <span class="ow">and</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">regex</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="n">f</span><span class="p">[</span><span class="s1">'path'</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="si">%s</span><span class="s1">/'</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="p">,</span> <span class="s1">''</span><span class="p">))]</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">filter_for_ignored_ext</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ignored_ext</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">ignore_copying</span><span class="p">)</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">filter_for_filesize</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">file_size</span><span class="p">)</span>
+ <span class="k">return</span> <span class="nb">bool</span><span class="p">(</span><span class="n">result</span><span class="p">)</span></div></div>
+
+
+<div class="viewcode-block" id="HdfsSensorFolder"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.sensors.hdfs_sensor.HdfsSensorFolder">[docs]</a><span class="k">class</span> <span class="nc">HdfsSensorFolder</span><span class="p">(</span><span class="n">HdfsSensor</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">be_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HdfsSensorFolder</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">be_empty</span> <span class="o">=</span> <span class="n">be_empty</span>
+
+<div class="viewcode-block" id="HdfsSensorFolder.poke"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.sensors.hdfs_sensor.HdfsSensorFolder.poke">[docs]</a> <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> poke for a non empty directory</span>
+
+<span class="sd"> :return: Bool depending on the search criteria</span>
+<span class="sd"> """</span>
+ <span class="n">sb</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="p">[</span><span class="n">f</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">sb</span><span class="o">.</span><span class="n">ls</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="p">],</span> <span class="n">include_toplevel</span><span class="o">=</span><span class="kc">True</span><span class="p">)]</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">filter_for_ignored_ext</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ignored_ext</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">ignore_copying</span><span class="p">)</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">filter_for_filesize</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">file_size</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">be_empty</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Poking for filepath </span><span class="si">{self.filepath}</span><span class="s1"> to a empty directory'</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
+ <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="n">result</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span> <span class="ow">and</span> <span class="n">result</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s1">'path'</span><span class="p">]</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">filepath</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Poking for filepath </span><span class="si">{self.filepath}</span><span class="s1"> to a non empty directory'</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
+ <span class="n">result</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
+ <span class="k">return</span> <span class="nb">bool</span><span class="p">(</span><span class="n">result</span><span class="p">)</span> <span class="ow">and</span> <span class="n">result</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s1">'file_type'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'f'</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/sensors/jira_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/sensors/jira_sensor.html b/_modules/airflow/contrib/sensors/jira_sensor.html
new file mode 100644
index 0000000..ae6ba2d
--- /dev/null
+++ b/_modules/airflow/contrib/sensors/jira_sensor.html
@@ -0,0 +1,382 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.sensors.jira_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.sensors.jira_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.sensors.jira_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">from</span> <span class="nn">jira.resources</span> <span class="k">import</span> <span class="n">Resource</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.operators.jira_operator</span> <span class="k">import</span> <span class="n">JIRAError</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.operators.jira_operator</span> <span class="k">import</span> <span class="n">JiraOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.sensors.base_sensor_operator</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="JiraSensor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.sensors.jira_sensor.JiraSensor">[docs]</a><span class="k">class</span> <span class="nc">JiraSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Monitors a jira ticket for any change.</span>
+
+<span class="sd"> :param jira_conn_id: reference to a pre-defined Jira Connection</span>
+<span class="sd"> :type jira_conn_id: str</span>
+<span class="sd"> :param method_name: method name from jira-python-sdk to be execute</span>
+<span class="sd"> :type method_name: str</span>
+<span class="sd"> :param method_params: parameters for the method method_name</span>
+<span class="sd"> :type method_params: dict</span>
+<span class="sd"> :param result_processor: function that return boolean and act as a sensor response</span>
+<span class="sd"> :type result_processor: function</span>
+<span class="sd"> """</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">jira_conn_id</span><span class="o">=</span><span class="s1">'jira_default'</span><span class="p">,</span>
+ <span class="n">method_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">method_params</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">result_processor</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">JiraSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">jira_conn_id</span> <span class="o">=</span> <span class="n">jira_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">result_processor</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="k">if</span> <span class="n">result_processor</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">result_processor</span> <span class="o">=</span> <span class="n">result_processor</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">method_name</span> <span class="o">=</span> <span class="n">method_name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">method_params</span> <span class="o">=</span> <span class="n">method_params</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">jira_operator</span> <span class="o">=</span> <span class="n">JiraOperator</span><span class="p">(</span><span class="n">task_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span>
+ <span class="n">jira_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">jira_conn_id</span><span class="p">,</span>
+ <span class="n">jira_method</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">method_name</span><span class="p">,</span>
+ <span class="n">jira_method_args</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">method_params</span><span class="p">,</span>
+ <span class="n">result_processor</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">result_processor</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">jira_operator</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">context</span><span class="o">=</span><span class="n">context</span><span class="p">)</span></div>
+
+
+<span class="k">class</span> <span class="nc">JiraTicketSensor</span><span class="p">(</span><span class="n">JiraSensor</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Monitors a jira ticket for given change in terms of function.</span>
+
+<span class="sd"> :param jira_conn_id: reference to a pre-defined Jira Connection</span>
+<span class="sd"> :type jira_conn_id: str</span>
+<span class="sd"> :param ticket_id: id of the ticket to be monitored</span>
+<span class="sd"> :type ticket_id: str</span>
+<span class="sd"> :param field: field of the ticket to be monitored</span>
+<span class="sd"> :type field: str</span>
+<span class="sd"> :param expected_value: expected value of the field</span>
+<span class="sd"> :type expected_value: str</span>
+<span class="sd"> :param result_processor: function that return boolean and act as a sensor response</span>
+<span class="sd"> :type result_processor: function</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s2">"ticket_id"</span><span class="p">,)</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">jira_conn_id</span><span class="o">=</span><span class="s1">'jira_default'</span><span class="p">,</span>
+ <span class="n">ticket_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">field</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">expected_value</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">field_checker_func</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">jira_conn_id</span> <span class="o">=</span> <span class="n">jira_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">ticket_id</span> <span class="o">=</span> <span class="n">ticket_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">field</span> <span class="o">=</span> <span class="n">field</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">expected_value</span> <span class="o">=</span> <span class="n">expected_value</span>
+ <span class="k">if</span> <span class="n">field_checker_func</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">field_checker_func</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">issue_field_checker</span>
+
+ <span class="nb">super</span><span class="p">(</span><span class="n">JiraTicketSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">jira_conn_id</span><span class="o">=</span><span class="n">jira_conn_id</span><span class="p">,</span>
+ <span class="n">result_processor</span><span class="o">=</span><span class="n">field_checker_func</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Jira Sensor checking for change in ticket: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ticket_id</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">jira_operator</span><span class="o">.</span><span class="n">method_name</span> <span class="o">=</span> <span class="s2">"issue"</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">jira_operator</span><span class="o">.</span><span class="n">jira_method_args</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'id'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">ticket_id</span><span class="p">,</span>
+ <span class="s1">'fields'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">field</span>
+ <span class="p">}</span>
+ <span class="k">return</span> <span class="n">JiraSensor</span><span class="o">.</span><span class="n">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="o">=</span><span class="n">context</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">issue_field_checker</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">,</span> <span class="n">issue</span><span class="p">):</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">issue</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> \
+ <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">field</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> \
+ <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">expected_value</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+
+ <span class="n">field_val</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">issue</span><span class="o">.</span><span class="n">fields</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">field</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">field_val</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">field_val</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">expected_value</span> <span class="ow">in</span> <span class="n">field_val</span>
+ <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">field_val</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">expected_value</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="n">field_val</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
+ <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">field_val</span><span class="p">,</span> <span class="n">Resource</span><span class="p">)</span> <span class="ow">and</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">field_val</span><span class="p">,</span> <span class="s1">'name'</span><span class="p">):</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">expected_value</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="n">field_val</span><span class="o">.</span><span class="n">name</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
+ <span class="s2">"Not implemented checker for issue field </span><span class="si">%s</span><span class="s2"> which "</span>
+ <span class="s2">"is neither string nor list nor Jira Resource"</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">field</span>
+ <span class="p">)</span>
+
+ <span class="k">except</span> <span class="n">JIRAError</span> <span class="k">as</span> <span class="n">jira_error</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Jira error while checking with expected value: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">jira_error</span><span class="p">)</span>
+ <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Error while checking with expected value </span><span class="si">%s</span><span class="s2">:"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">expected_value</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">result</span> <span class="ow">is</span> <span class="kc">True</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Issue field </span><span class="si">%s</span><span class="s2"> has expected value </span><span class="si">%s</span><span class="s2">, returning success"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">field</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">expected_value</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Issue field </span><span class="si">%s</span><span class="s2"> don't have expected value </span><span class="si">%s</span><span class="s2"> yet."</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">field</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">expected_value</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">result</span>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/sensors/pubsub_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/sensors/pubsub_sensor.html b/_modules/airflow/contrib/sensors/pubsub_sensor.html
new file mode 100644
index 0000000..165e293
--- /dev/null
+++ b/_modules/airflow/contrib/sensors/pubsub_sensor.html
@@ -0,0 +1,340 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.sensors.pubsub_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.sensors.pubsub_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.sensors.pubsub_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_pubsub_hook</span> <span class="k">import</span> <span class="n">PubSubHook</span>
+<span class="kn">from</span> <span class="nn">airflow.sensors.base_sensor_operator</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="PubSubPullSensor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.sensors.pubsub_sensor.PubSubPullSensor">[docs]</a><span class="k">class</span> <span class="nc">PubSubPullSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""Pulls messages from a PubSub subscription and passes them through XCom.</span>
+
+<span class="sd"> This sensor operator will pull up to ``max_messages`` messages from the</span>
+<span class="sd"> specified PubSub subscription. When the subscription returns messages,</span>
+<span class="sd"> the poke method's criteria will be fulfilled and the messages will be</span>
+<span class="sd"> returned from the operator and passed through XCom for downstream tasks.</span>
+
+<span class="sd"> If ``ack_messages`` is set to True, messages will be immediately</span>
+<span class="sd"> acknowledged before being returned, otherwise, downstream tasks will be</span>
+<span class="sd"> responsible for acknowledging them.</span>
+
+<span class="sd"> ``project`` and ``subscription`` are templated so you can use</span>
+<span class="sd"> variables in them.</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'project'</span><span class="p">,</span> <span class="s1">'subscription'</span><span class="p">]</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#ff7f50'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">project</span><span class="p">,</span>
+ <span class="n">subscription</span><span class="p">,</span>
+ <span class="n">max_messages</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span>
+ <span class="n">return_immediately</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">ack_messages</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> :param project: the GCP project ID for the subscription (templated)</span>
+<span class="sd"> :type project: string</span>
+<span class="sd"> :param subscription: the Pub/Sub subscription name. Do not include the</span>
+<span class="sd"> full subscription path.</span>
+<span class="sd"> :type subscription: string</span>
+<span class="sd"> :param max_messages: The maximum number of messages to retrieve per</span>
+<span class="sd"> PubSub pull request</span>
+<span class="sd"> :type max_messages: int</span>
+<span class="sd"> :param return_immediately: If True, instruct the PubSub API to return</span>
+<span class="sd"> immediately if no messages are available for delivery.</span>
+<span class="sd"> :type return_immediately: bool</span>
+<span class="sd"> :param ack_messages: If True, each message will be acknowledged</span>
+<span class="sd"> immediately rather than by any downstream tasks</span>
+<span class="sd"> :type ack_messages: bool</span>
+<span class="sd"> :param gcp_conn_id: The connection ID to use connecting to</span>
+<span class="sd"> Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request</span>
+<span class="sd"> must have domain-wide delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> """</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">PubSubPullSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">project</span> <span class="o">=</span> <span class="n">project</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">subscription</span> <span class="o">=</span> <span class="n">subscription</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">max_messages</span> <span class="o">=</span> <span class="n">max_messages</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">return_immediately</span> <span class="o">=</span> <span class="n">return_immediately</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">ack_messages</span> <span class="o">=</span> <span class="n">ack_messages</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">_messages</span> <span class="o">=</span> <span class="kc">None</span>
+
+<div class="viewcode-block" id="PubSubPullSensor.execute"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.sensors.pubsub_sensor.PubSubPullSensor.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="sd">"""Overridden to allow messages to be passed"""</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">PubSubPullSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">context</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_messages</span></div>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">PubSubHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_messages</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">pull</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">project</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">subscription</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_messages</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">return_immediately</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_messages</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">ack_messages</span><span class="p">:</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">ack_messages</span><span class="p">:</span>
+ <span class="n">ack_ids</span> <span class="o">=</span> <span class="p">[</span><span class="n">m</span><span class="p">[</span><span class="s1">'ackId'</span><span class="p">]</span> <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_messages</span> <span class="k">if</span> <span class="n">m</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'ackId'</span><span class="p">)]</span>
+ <span class="n">hook</span><span class="o">.</span><span class="n">acknowledge</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">subscription</span><span class="p">,</span> <span class="n">ack_ids</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_messages</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/sensors/qubole_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/sensors/qubole_sensor.html b/_modules/airflow/contrib/sensors/qubole_sensor.html
new file mode 100644
index 0000000..e7becab
--- /dev/null
+++ b/_modules/airflow/contrib/sensors/qubole_sensor.html
@@ -0,0 +1,323 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.sensors.qubole_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.sensors.qubole_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.sensors.qubole_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">qds_sdk.qubole</span> <span class="k">import</span> <span class="n">Qubole</span>
+<span class="kn">from</span> <span class="nn">qds_sdk.sensors</span> <span class="k">import</span> <span class="n">FileSensor</span><span class="p">,</span> <span class="n">PartitionSensor</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.sensors.base_sensor_operator</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="QuboleSensor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.sensors.qubole_sensor.QuboleSensor">[docs]</a><span class="k">class</span> <span class="nc">QuboleSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Base class for all Qubole Sensors</span>
+
+<span class="sd"> :param qubole_conn_id: The qubole connection to run the sensor against</span>
+<span class="sd"> :type qubole_conn_id: string</span>
+<span class="sd"> :param data: a JSON object containing payload, whose presence needs to be checked</span>
+<span class="sd"> :type data: a JSON object</span>
+
+<span class="sd"> .. note:: Both ``data`` and ``qubole_conn_id`` fields are template-supported. You can</span>
+<span class="sd"> also use ``.txt`` files for template driven use cases.</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'data'</span><span class="p">,</span> <span class="s1">'qubole_conn_id'</span><span class="p">)</span>
+
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.txt'</span><span class="p">,)</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">qubole_conn_id</span><span class="o">=</span><span class="s2">"qubole_default"</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">data</span> <span class="o">=</span> <span class="n">data</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">qubole_conn_id</span> <span class="o">=</span> <span class="n">qubole_conn_id</span>
+
+ <span class="k">if</span> <span class="s1">'poke_interval'</span> <span class="ow">in</span> <span class="n">kwargs</span> <span class="ow">and</span> <span class="n">kwargs</span><span class="p">[</span><span class="s1">'poke_interval'</span><span class="p">]</span> <span class="o"><</span> <span class="mi">5</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Sorry, poke_interval can't be less than 5 sec for "</span>
+ <span class="s2">"task '</span><span class="si">{0}</span><span class="s2">' in dag '</span><span class="si">{1}</span><span class="s2">'."</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">kwargs</span><span class="p">[</span><span class="s1">'task_id'</span><span class="p">],</span> <span class="n">kwargs</span><span class="p">[</span><span class="s1">'dag'</span><span class="p">]</span><span class="o">.</span><span class="n">dag_id</span><span class="p">))</span>
+
+ <span class="nb">super</span><span class="p">(</span><span class="n">QuboleSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="k">global</span> <span class="n">this</span> <span class="c1"># apache/incubator-airflow/pull/3297#issuecomment-385988083</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="n">BaseHook</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">qubole_conn_id</span><span class="p">)</span>
+ <span class="n">Qubole</span><span class="o">.</span><span class="n">configure</span><span class="p">(</span><span class="n">api_token</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span> <span class="n">api_url</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">)</span>
+
+ <span class="n">this</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Poking: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
+
+ <span class="n">status</span> <span class="o">=</span> <span class="kc">False</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">status</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">sensor_class</span><span class="o">.</span><span class="n">check</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
+ <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="n">this</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
+ <span class="n">status</span> <span class="o">=</span> <span class="kc">False</span>
+
+ <span class="n">this</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Status of this Poke: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">status</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">status</span></div>
+
+
+<span class="k">class</span> <span class="nc">QuboleFileSensor</span><span class="p">(</span><span class="n">QuboleSensor</span><span class="p">):</span>
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sensor_class</span> <span class="o">=</span> <span class="n">FileSensor</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">QuboleFileSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">QubolePartitionSensor</span><span class="p">(</span><span class="n">QuboleSensor</span><span class="p">):</span>
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sensor_class</span> <span class="o">=</span> <span class="n">PartitionSensor</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">QubolePartitionSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[33/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/dataproc_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/dataproc_operator.html b/_modules/airflow/contrib/operators/dataproc_operator.html
index d49719e..f26e459 100644
--- a/_modules/airflow/contrib/operators/dataproc_operator.html
+++ b/_modules/airflow/contrib/operators/dataproc_operator.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -169,29 +171,42 @@
<h1>Source code for airflow.contrib.operators.dataproc_operator</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="c1">#</span>
+<span class="kn">import</span> <span class="nn">ntpath</span>
+<span class="kn">import</span> <span class="nn">os</span>
+<span class="kn">import</span> <span class="nn">re</span>
<span class="kn">import</span> <span class="nn">time</span>
+<span class="kn">import</span> <span class="nn">uuid</span>
+<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">timedelta</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_dataproc_hook</span> <span class="k">import</span> <span class="n">DataProcHook</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcs_hook</span> <span class="k">import</span> <span class="n">GoogleCloudStorageHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
<span class="kn">from</span> <span class="nn">airflow.version</span> <span class="k">import</span> <span class="n">version</span>
<span class="kn">from</span> <span class="nn">googleapiclient.errors</span> <span class="k">import</span> <span class="n">HttpError</span>
+<span class="kn">from</span> <span class="nn">airflow.utils</span> <span class="k">import</span> <span class="n">timezone</span>
-<span class="k">class</span> <span class="nc">DataprocClusterCreateOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+<div class="viewcode-block" id="DataprocClusterCreateOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataprocClusterCreateOperator">[docs]</a><span class="k">class</span> <span class="nc">DataprocClusterCreateOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Create a new cluster on Google Cloud Dataproc. The operator will wait until the</span>
<span class="sd"> creation is successful or an error occurs in the creation process.</span>
@@ -202,9 +217,79 @@
<span class="sd"> for a detailed explanation on the different parameters. Most of the configuration</span>
<span class="sd"> parameters detailed in the link are available as a parameter to this operator.</span>
+
+<span class="sd"> :param cluster_name: The name of the DataProc cluster to create.</span>
+<span class="sd"> :type cluster_name: string</span>
+<span class="sd"> :param project_id: The ID of the google cloud project in which</span>
+<span class="sd"> to create the cluster</span>
+<span class="sd"> :type project_id: string</span>
+<span class="sd"> :param num_workers: The # of workers to spin up</span>
+<span class="sd"> :type num_workers: int</span>
+<span class="sd"> :param storage_bucket: The storage bucket to use, setting to None lets dataproc</span>
+<span class="sd"> generate a custom one for you</span>
+<span class="sd"> :type storage_bucket: string</span>
+<span class="sd"> :param init_actions_uris: List of GCS uri's containing</span>
+<span class="sd"> dataproc initialization scripts</span>
+<span class="sd"> :type init_actions_uris: list[string]</span>
+<span class="sd"> :param init_action_timeout: Amount of time executable scripts in</span>
+<span class="sd"> init_actions_uris has to complete</span>
+<span class="sd"> :type init_action_timeout: string</span>
+<span class="sd"> :param metadata: dict of key-value google compute engine metadata entries</span>
+<span class="sd"> to add to all instances</span>
+<span class="sd"> :type metadata: dict</span>
+<span class="sd"> :param image_version: the version of software inside the Dataproc cluster</span>
+<span class="sd"> :type image_version: string</span>
+<span class="sd"> :param properties: dict of properties to set on</span>
+<span class="sd"> config files (e.g. spark-defaults.conf), see</span>
+<span class="sd"> https://cloud.google.com/dataproc/docs/reference/rest/v1/ \</span>
+<span class="sd"> projects.regions.clusters#SoftwareConfig</span>
+<span class="sd"> :type properties: dict</span>
+<span class="sd"> :param master_machine_type: Compute engine machine type to use for the master node</span>
+<span class="sd"> :type master_machine_type: string</span>
+<span class="sd"> :param master_disk_size: Disk size for the master node</span>
+<span class="sd"> :type master_disk_size: int</span>
+<span class="sd"> :param worker_machine_type: Compute engine machine type to use for the worker nodes</span>
+<span class="sd"> :type worker_machine_type: string</span>
+<span class="sd"> :param worker_disk_size: Disk size for the worker nodes</span>
+<span class="sd"> :type worker_disk_size: int</span>
+<span class="sd"> :param num_preemptible_workers: The # of preemptible worker nodes to spin up</span>
+<span class="sd"> :type num_preemptible_workers: int</span>
+<span class="sd"> :param labels: dict of labels to add to the cluster</span>
+<span class="sd"> :type labels: dict</span>
+<span class="sd"> :param zone: The zone where the cluster will be located</span>
+<span class="sd"> :type zone: string</span>
+<span class="sd"> :param network_uri: The network uri to be used for machine communication, cannot be</span>
+<span class="sd"> specified with subnetwork_uri</span>
+<span class="sd"> :type network_uri: string</span>
+<span class="sd"> :param subnetwork_uri: The subnetwork uri to be used for machine communication,</span>
+<span class="sd"> cannot be specified with network_uri</span>
+<span class="sd"> :type subnetwork_uri: string</span>
+<span class="sd"> :param tags: The GCE tags to add to all instances</span>
+<span class="sd"> :type tags: list[string]</span>
+<span class="sd"> :param region: leave as 'global', might become relevant in the future</span>
+<span class="sd"> :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request must have domain-wide</span>
+<span class="sd"> delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> :param service_account: The service account of the dataproc instances.</span>
+<span class="sd"> :type service_account: string</span>
+<span class="sd"> :param service_account_scopes: The URIs of service account scopes to be included.</span>
+<span class="sd"> :type service_account_scopes: list[string]</span>
+<span class="sd"> :param idle_delete_ttl: The longest duration that cluster would keep alive while</span>
+<span class="sd"> staying idle. Passing this threshold will cause cluster to be auto-deleted.</span>
+<span class="sd"> A duration in seconds.</span>
+<span class="sd"> :type idle_delete_ttl: int</span>
+<span class="sd"> :param auto_delete_time: The time when cluster will be auto-deleted.</span>
+<span class="sd"> :type auto_delete_time: datetime</span>
+<span class="sd"> :param auto_delete_ttl: The life duration of cluster, the cluster will be</span>
+<span class="sd"> auto-deleted at the end of this duration.</span>
+<span class="sd"> A duration in seconds. (If auto_delete_time is set this parameter will be ignored)</span>
+<span class="sd"> :type auto_delete_ttl: int</span>
<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'cluster_name'</span><span class="p">,]</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'cluster_name'</span><span class="p">,</span> <span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'zone'</span><span class="p">,</span> <span class="s1">'region'</span><span class="p">]</span>
<span class="nd">@apply_defaults</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
@@ -212,8 +297,12 @@
<span class="n">project_id</span><span class="p">,</span>
<span class="n">num_workers</span><span class="p">,</span>
<span class="n">zone</span><span class="p">,</span>
+ <span class="n">network_uri</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">subnetwork_uri</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">tags</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">storage_bucket</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">init_actions_uris</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">init_action_timeout</span><span class="o">=</span><span class="s2">"10m"</span><span class="p">,</span>
<span class="n">metadata</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">image_version</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">properties</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -228,64 +317,12 @@
<span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">service_account</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">service_account_scopes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">idle_delete_ttl</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">auto_delete_time</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">auto_delete_ttl</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span>
<span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Create a new DataprocClusterCreateOperator.</span>
-
-<span class="sd"> For more info on the creation of a cluster through the API, have a look at:</span>
-
-<span class="sd"> https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.clusters</span>
-
-<span class="sd"> :param cluster_name: The name of the DataProc cluster to create.</span>
-<span class="sd"> :type cluster_name: string</span>
-<span class="sd"> :param project_id: The ID of the google cloud project in which</span>
-<span class="sd"> to create the cluster</span>
-<span class="sd"> :type project_id: string</span>
-<span class="sd"> :param num_workers: The # of workers to spin up</span>
-<span class="sd"> :type num_workers: int</span>
-<span class="sd"> :param storage_bucket: The storage bucket to use, setting to None lets dataproc</span>
-<span class="sd"> generate a custom one for you</span>
-<span class="sd"> :type storage_bucket: string</span>
-<span class="sd"> :param init_actions_uris: List of GCS uri's containing</span>
-<span class="sd"> dataproc initialization scripts</span>
-<span class="sd"> :type init_actions_uris: list[string]</span>
-<span class="sd"> :param metadata: dict of key-value google compute engine metadata entries</span>
-<span class="sd"> to add to all instances</span>
-<span class="sd"> :type metadata: dict</span>
-<span class="sd"> :param image_version: the version of software inside the Dataproc cluster</span>
-<span class="sd"> :type image_version: string</span>
-<span class="sd"> :param properties: dict of properties to set on</span>
-<span class="sd"> config files (e.g. spark-defaults.conf), see</span>
-<span class="sd"> https://cloud.google.com/dataproc/docs/reference/rest/v1/ \</span>
-<span class="sd"> projects.regions.clusters#SoftwareConfig</span>
-<span class="sd"> :type properties: dict</span>
-<span class="sd"> :param master_machine_type: Compute engine machine type to use for the master node</span>
-<span class="sd"> :type master_machine_type: string</span>
-<span class="sd"> :param master_disk_size: Disk size for the master node</span>
-<span class="sd"> :type int</span>
-<span class="sd"> :param worker_machine_type:Compute engine machine type to use for the worker nodes</span>
-<span class="sd"> :type worker_machine_type: string</span>
-<span class="sd"> :param worker_disk_size: Disk size for the worker nodes</span>
-<span class="sd"> :type worker_disk_size: int</span>
-<span class="sd"> :param num_preemptible_workers: The # of preemptible worker nodes to spin up</span>
-<span class="sd"> :type num_preemptible_workers: int</span>
-<span class="sd"> :param labels: dict of labels to add to the cluster</span>
-<span class="sd"> :type labels: dict</span>
-<span class="sd"> :param zone: The zone where the cluster will be located</span>
-<span class="sd"> :type zone: string</span>
-<span class="sd"> :param region: leave as 'global', might become relevant in the future</span>
-<span class="sd"> :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.</span>
-<span class="sd"> :type gcp_conn_id: string</span>
-<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
-<span class="sd"> For this to work, the service account making the request must have domain-wide</span>
-<span class="sd"> delegation enabled.</span>
-<span class="sd"> :type delegate_to: string</span>
-<span class="sd"> :param service_account: The service account of the dataproc instances.</span>
-<span class="sd"> :type service_account: string</span>
-<span class="sd"> :param service_account_scopes: The URIs of service account scopes to be included.</span>
-<span class="sd"> :type service_account_scopes: list[string]</span>
-<span class="sd"> """</span>
+
<span class="nb">super</span><span class="p">(</span><span class="n">DataprocClusterCreateOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
@@ -295,6 +332,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">num_preemptible_workers</span> <span class="o">=</span> <span class="n">num_preemptible_workers</span>
<span class="bp">self</span><span class="o">.</span><span class="n">storage_bucket</span> <span class="o">=</span> <span class="n">storage_bucket</span>
<span class="bp">self</span><span class="o">.</span><span class="n">init_actions_uris</span> <span class="o">=</span> <span class="n">init_actions_uris</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">init_action_timeout</span> <span class="o">=</span> <span class="n">init_action_timeout</span>
<span class="bp">self</span><span class="o">.</span><span class="n">metadata</span> <span class="o">=</span> <span class="n">metadata</span>
<span class="bp">self</span><span class="o">.</span><span class="n">image_version</span> <span class="o">=</span> <span class="n">image_version</span>
<span class="bp">self</span><span class="o">.</span><span class="n">properties</span> <span class="o">=</span> <span class="n">properties</span>
@@ -304,9 +342,15 @@
<span class="bp">self</span><span class="o">.</span><span class="n">worker_disk_size</span> <span class="o">=</span> <span class="n">worker_disk_size</span>
<span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="o">=</span> <span class="n">labels</span>
<span class="bp">self</span><span class="o">.</span><span class="n">zone</span> <span class="o">=</span> <span class="n">zone</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">network_uri</span> <span class="o">=</span> <span class="n">network_uri</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">subnetwork_uri</span> <span class="o">=</span> <span class="n">subnetwork_uri</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">tags</span> <span class="o">=</span> <span class="n">tags</span>
<span class="bp">self</span><span class="o">.</span><span class="n">region</span> <span class="o">=</span> <span class="n">region</span>
<span class="bp">self</span><span class="o">.</span><span class="n">service_account</span> <span class="o">=</span> <span class="n">service_account</span>
<span class="bp">self</span><span class="o">.</span><span class="n">service_account_scopes</span> <span class="o">=</span> <span class="n">service_account_scopes</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">idle_delete_ttl</span> <span class="o">=</span> <span class="n">idle_delete_ttl</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">auto_delete_time</span> <span class="o">=</span> <span class="n">auto_delete_time</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">auto_delete_ttl</span> <span class="o">=</span> <span class="n">auto_delete_ttl</span>
<span class="k">def</span> <span class="nf">_get_cluster_list_for_project</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">service</span><span class="p">):</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">service</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">regions</span><span class="p">()</span><span class="o">.</span><span class="n">clusters</span><span class="p">()</span><span class="o">.</span><span class="n">list</span><span class="p">(</span>
@@ -357,6 +401,19 @@
<span class="k">return</span>
<span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">15</span><span class="p">)</span>
+ <span class="k">def</span> <span class="nf">_get_init_action_timeout</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="n">match</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="sa">r</span><span class="s2">"^(\d+)(s|m)$"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">init_action_timeout</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">match</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span> <span class="o">==</span> <span class="s2">"s"</span><span class="p">:</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">init_action_timeout</span>
+ <span class="k">elif</span> <span class="n">match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span> <span class="o">==</span> <span class="s2">"m"</span><span class="p">:</span>
+ <span class="n">val</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span>
+ <span class="k">return</span> <span class="s2">"</span><span class="si">{}</span><span class="s2">s"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">timedelta</span><span class="p">(</span><span class="n">minutes</span><span class="o">=</span><span class="n">val</span><span class="p">)</span><span class="o">.</span><span class="n">seconds</span><span class="p">)</span>
+
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s2">"DataprocClusterCreateOperator init_action_timeout"</span>
+ <span class="s2">" should be expressed in minutes or seconds. i.e. 10m, 30s"</span><span class="p">)</span>
+
<span class="k">def</span> <span class="nf">_build_cluster_data</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">zone_uri</span> <span class="o">=</span> \
<span class="s1">'https://www.googleapis.com/compute/v1/projects/</span><span class="si">{}</span><span class="s1">/zones/</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
@@ -392,7 +449,8 @@
<span class="p">}</span>
<span class="p">},</span>
<span class="s1">'secondaryWorkerConfig'</span><span class="p">:</span> <span class="p">{},</span>
- <span class="s1">'softwareConfig'</span><span class="p">:</span> <span class="p">{}</span>
+ <span class="s1">'softwareConfig'</span><span class="p">:</span> <span class="p">{},</span>
+ <span class="s1">'lifecycleConfig'</span><span class="p">:</span> <span class="p">{}</span>
<span class="p">}</span>
<span class="p">}</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_preemptible_workers</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
@@ -410,18 +468,37 @@
<span class="c1"># [a-z]([-a-z0-9]*[a-z0-9])? (current airflow version string follows</span>
<span class="c1"># semantic versioning spec: x.y.z).</span>
<span class="n">cluster_data</span><span class="p">[</span><span class="s1">'labels'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s1">'airflow-version'</span><span class="p">:</span>
- <span class="s1">'v'</span> <span class="o">+</span> <span class="n">version</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'.'</span><span class="p">,</span> <span class="s1">'-'</span><span class="p">)})</span>
+ <span class="s1">'v'</span> <span class="o">+</span> <span class="n">version</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'.'</span><span class="p">,</span> <span class="s1">'-'</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'+'</span><span class="p">,</span><span class="s1">'-'</span><span class="p">)})</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">storage_bucket</span><span class="p">:</span>
<span class="n">cluster_data</span><span class="p">[</span><span class="s1">'config'</span><span class="p">][</span><span class="s1">'configBucket'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">storage_bucket</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="p">:</span>
<span class="n">cluster_data</span><span class="p">[</span><span class="s1">'config'</span><span class="p">][</span><span class="s1">'gceClusterConfig'</span><span class="p">][</span><span class="s1">'metadata'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">network_uri</span><span class="p">:</span>
+ <span class="n">cluster_data</span><span class="p">[</span><span class="s1">'config'</span><span class="p">][</span><span class="s1">'gceClusterConfig'</span><span class="p">][</span><span class="s1">'networkUri'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">network_uri</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">subnetwork_uri</span><span class="p">:</span>
+ <span class="n">cluster_data</span><span class="p">[</span><span class="s1">'config'</span><span class="p">][</span><span class="s1">'gceClusterConfig'</span><span class="p">][</span><span class="s1">'subnetworkUri'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">subnetwork_uri</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">tags</span><span class="p">:</span>
+ <span class="n">cluster_data</span><span class="p">[</span><span class="s1">'config'</span><span class="p">][</span><span class="s1">'gceClusterConfig'</span><span class="p">][</span><span class="s1">'tags'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">tags</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">image_version</span><span class="p">:</span>
<span class="n">cluster_data</span><span class="p">[</span><span class="s1">'config'</span><span class="p">][</span><span class="s1">'softwareConfig'</span><span class="p">][</span><span class="s1">'imageVersion'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">image_version</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">properties</span><span class="p">:</span>
<span class="n">cluster_data</span><span class="p">[</span><span class="s1">'config'</span><span class="p">][</span><span class="s1">'softwareConfig'</span><span class="p">][</span><span class="s1">'properties'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">properties</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">idle_delete_ttl</span><span class="p">:</span>
+ <span class="n">cluster_data</span><span class="p">[</span><span class="s1">'config'</span><span class="p">][</span><span class="s1">'lifecycleConfig'</span><span class="p">][</span><span class="s1">'idleDeleteTtl'</span><span class="p">]</span> <span class="o">=</span> \
+ <span class="s2">"</span><span class="si">{}</span><span class="s2">s"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">idle_delete_ttl</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">auto_delete_time</span><span class="p">:</span>
+ <span class="n">utc_auto_delete_time</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">convert_to_utc</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">auto_delete_time</span><span class="p">)</span>
+ <span class="n">cluster_data</span><span class="p">[</span><span class="s1">'config'</span><span class="p">][</span><span class="s1">'lifecycleConfig'</span><span class="p">][</span><span class="s1">'autoDeleteTime'</span><span class="p">]</span> <span class="o">=</span> \
+ <span class="n">utc_auto_delete_time</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="s1">'%Y-%m-</span><span class="si">%d</span><span class="s1">T%H:%M:%S.</span><span class="si">%f</span><span class="s1">Z'</span><span class="p">,</span> <span class="n">formatter</span><span class="o">=</span><span class="s1">'classic'</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">auto_delete_ttl</span><span class="p">:</span>
+ <span class="n">cluster_data</span><span class="p">[</span><span class="s1">'config'</span><span class="p">][</span><span class="s1">'lifecycleConfig'</span><span class="p">][</span><span class="s1">'autoDeleteTtl'</span><span class="p">]</span> <span class="o">=</span> \
+ <span class="s2">"</span><span class="si">{}</span><span class="s2">s"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">auto_delete_ttl</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">init_actions_uris</span><span class="p">:</span>
<span class="n">init_actions_dict</span> <span class="o">=</span> <span class="p">[</span>
- <span class="p">{</span><span class="s1">'executableFile'</span><span class="p">:</span> <span class="n">uri</span><span class="p">}</span> <span class="k">for</span> <span class="n">uri</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">init_actions_uris</span>
+ <span class="p">{</span>
+ <span class="s1">'executableFile'</span><span class="p">:</span> <span class="n">uri</span><span class="p">,</span>
+ <span class="s1">'executionTimeout'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_init_action_timeout</span><span class="p">()</span>
+ <span class="p">}</span> <span class="k">for</span> <span class="n">uri</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">init_actions_uris</span>
<span class="p">]</span>
<span class="n">cluster_data</span><span class="p">[</span><span class="s1">'config'</span><span class="p">][</span><span class="s1">'initializationActions'</span><span class="p">]</span> <span class="o">=</span> <span class="n">init_actions_dict</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">service_account</span><span class="p">:</span>
@@ -468,16 +545,30 @@
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">e</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_done</span><span class="p">(</span><span class="n">service</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_done</span><span class="p">(</span><span class="n">service</span><span class="p">)</span></div>
-<span class="k">class</span> <span class="nc">DataprocClusterDeleteOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+<div class="viewcode-block" id="DataprocClusterDeleteOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataprocClusterDeleteOperator">[docs]</a><span class="k">class</span> <span class="nc">DataprocClusterDeleteOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Delete a cluster on Google Cloud Dataproc. The operator will wait until the</span>
<span class="sd"> cluster is destroyed.</span>
+
+<span class="sd"> :param cluster_name: The name of the cluster to create.</span>
+<span class="sd"> :type cluster_name: string</span>
+<span class="sd"> :param project_id: The ID of the google cloud project in which</span>
+<span class="sd"> the cluster runs</span>
+<span class="sd"> :type project_id: string</span>
+<span class="sd"> :param region: leave as 'global', might become relevant in the future</span>
+<span class="sd"> :type region: string</span>
+<span class="sd"> :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request must have domain-wide</span>
+<span class="sd"> delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'cluster_name'</span><span class="p">]</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'cluster_name'</span><span class="p">,</span> <span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'region'</span><span class="p">]</span>
<span class="nd">@apply_defaults</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
@@ -488,23 +579,7 @@
<span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span>
<span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Delete a cluster on Google Cloud Dataproc.</span>
-
-<span class="sd"> :param cluster_name: The name of the cluster to create.</span>
-<span class="sd"> :type cluster_name: string</span>
-<span class="sd"> :param project_id: The ID of the google cloud project in which</span>
-<span class="sd"> the cluster runs</span>
-<span class="sd"> :type project_id: string</span>
-<span class="sd"> :param region: leave as 'global', might become relevant in the future</span>
-<span class="sd"> :type region: string</span>
-<span class="sd"> :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.</span>
-<span class="sd"> :type gcp_conn_id: string</span>
-<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
-<span class="sd"> For this to work, the service account making the request must have domain-wide</span>
-<span class="sd"> delegation enabled.</span>
-<span class="sd"> :type delegate_to: string</span>
-<span class="sd"> """</span>
+
<span class="nb">super</span><span class="p">(</span><span class="n">DataprocClusterDeleteOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
@@ -541,7 +616,7 @@
<span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
<span class="n">operation_name</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Cluster delete operation name: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_done</span><span class="p">(</span><span class="n">service</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_done</span><span class="p">(</span><span class="n">service</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">)</span></div>
<div class="viewcode-block" id="DataProcPigOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcPigOperator">[docs]</a><span class="k">class</span> <span class="nc">DataProcPigOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
@@ -552,29 +627,60 @@
<span class="sd"> It's a good practice to define dataproc_* parameters in the default_args of the dag</span>
<span class="sd"> like the cluster name and UDFs.</span>
-<span class="sd"> ```</span>
-<span class="sd"> default_args = {</span>
-<span class="sd"> 'cluster_name': 'cluster-1',</span>
-<span class="sd"> 'dataproc_pig_jars': [</span>
-<span class="sd"> 'gs://example/udf/jar/datafu/1.2.0/datafu.jar',</span>
-<span class="sd"> 'gs://example/udf/jar/gpig/1.2/gpig.jar'</span>
-<span class="sd"> ]</span>
-<span class="sd"> }</span>
-<span class="sd"> ```</span>
+<span class="sd"> .. code-block:: python</span>
+
+<span class="sd"> default_args = {</span>
+<span class="sd"> 'cluster_name': 'cluster-1',</span>
+<span class="sd"> 'dataproc_pig_jars': [</span>
+<span class="sd"> 'gs://example/udf/jar/datafu/1.2.0/datafu.jar',</span>
+<span class="sd"> 'gs://example/udf/jar/gpig/1.2/gpig.jar'</span>
+<span class="sd"> ]</span>
+<span class="sd"> }</span>
<span class="sd"> You can pass a pig script as string or file reference. Use variables to pass on</span>
<span class="sd"> variables for the pig script to be resolved on the cluster or use the parameters to</span>
<span class="sd"> be resolved in the script as template parameters.</span>
-<span class="sd"> ```</span>
-<span class="sd"> t1 = DataProcPigOperator(</span>
-<span class="sd"> task_id='dataproc_pig',</span>
-<span class="sd"> query='a_pig_script.pig',</span>
-<span class="sd"> variables={'out': 'gs://example/output/{{ds}}'},</span>
-<span class="sd"> dag=dag)</span>
-<span class="sd"> ```</span>
+<span class="sd"> **Example**: ::</span>
+
+<span class="sd"> t1 = DataProcPigOperator(</span>
+<span class="sd"> task_id='dataproc_pig',</span>
+<span class="sd"> query='a_pig_script.pig',</span>
+<span class="sd"> variables={'out': 'gs://example/output/{{ds}}'},</span>
+<span class="sd"> dag=dag)</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> For more detail on about job submission have a look at the reference:</span>
+<span class="sd"> https://cloud.google.com/dataproc/reference/rest/v1/projects.regions.jobs</span>
+
+<span class="sd"> :param query: The query or reference to the query file (pg or pig extension).</span>
+<span class="sd"> :type query: string</span>
+<span class="sd"> :param query_uri: The uri of a pig script on Cloud Storage.</span>
+<span class="sd"> :type query_uri: string</span>
+<span class="sd"> :param variables: Map of named parameters for the query.</span>
+<span class="sd"> :type variables: dict</span>
+<span class="sd"> :param job_name: The job name used in the DataProc cluster. This name by default</span>
+<span class="sd"> is the task_id appended with the execution data, but can be templated. The</span>
+<span class="sd"> name will always be appended with a random number to avoid name clashes.</span>
+<span class="sd"> :type job_name: string</span>
+<span class="sd"> :param cluster_name: The name of the DataProc cluster.</span>
+<span class="sd"> :type cluster_name: string</span>
+<span class="sd"> :param dataproc_pig_properties: Map for the Pig properties. Ideal to put in</span>
+<span class="sd"> default arguments</span>
+<span class="sd"> :type dataproc_pig_properties: dict</span>
+<span class="sd"> :param dataproc_pig_jars: URIs to jars provisioned in Cloud Storage (example: for</span>
+<span class="sd"> UDFs and libs) and are ideal to put in default arguments.</span>
+<span class="sd"> :type dataproc_pig_jars: list</span>
+<span class="sd"> :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request must have domain-wide</span>
+<span class="sd"> delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> :param region: The specified region where the dataproc cluster is created.</span>
+<span class="sd"> :type region: string</span>
<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'query'</span><span class="p">,</span> <span class="s1">'variables'</span><span class="p">,</span> <span class="s1">'job_name'</span><span class="p">,</span> <span class="s1">'cluster_name'</span><span class="p">]</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'query'</span><span class="p">,</span> <span class="s1">'variables'</span><span class="p">,</span> <span class="s1">'job_name'</span><span class="p">,</span> <span class="s1">'cluster_name'</span><span class="p">,</span> <span class="s1">'dataproc_jars'</span><span class="p">]</span>
<span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.pg'</span><span class="p">,</span> <span class="s1">'.pig'</span><span class="p">,)</span>
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#0273d4'</span>
@@ -590,40 +696,10 @@
<span class="n">dataproc_pig_jars</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">region</span><span class="o">=</span><span class="s1">'global'</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span>
<span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Create a new DataProcPigOperator.</span>
-
-<span class="sd"> For more detail on about job submission have a look at the reference:</span>
-
-<span class="sd"> https://cloud.google.com/dataproc/reference/rest/v1/projects.regions.jobs</span>
-<span class="sd"> :param query: The query or reference to the query file (pg or pig extension).</span>
-<span class="sd"> :type query: string</span>
-<span class="sd"> :param query_uri: The uri of a pig script on Cloud Storage.</span>
-<span class="sd"> :type query_uri: string</span>
-<span class="sd"> :param variables: Map of named parameters for the query.</span>
-<span class="sd"> :type variables: dict</span>
-<span class="sd"> :param job_name: The job name used in the DataProc cluster. This name by default</span>
-<span class="sd"> is the task_id appended with the execution data, but can be templated. The</span>
-<span class="sd"> name will always be appended with a random number to avoid name clashes.</span>
-<span class="sd"> :type job_name: string</span>
-<span class="sd"> :param cluster_name: The name of the DataProc cluster.</span>
-<span class="sd"> :type cluster_name: string</span>
-<span class="sd"> :param dataproc_pig_properties: Map for the Pig properties. Ideal to put in</span>
-<span class="sd"> default arguments</span>
-<span class="sd"> :type dataproc_pig_properties: dict</span>
-<span class="sd"> :param dataproc_pig_jars: URIs to jars provisioned in Cloud Storage (example: for</span>
-<span class="sd"> UDFs and libs) and are ideal to put in default arguments.</span>
-<span class="sd"> :type dataproc_pig_jars: list</span>
-<span class="sd"> :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.</span>
-<span class="sd"> :type gcp_conn_id: string</span>
-<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
-<span class="sd"> For this to work, the service account making the request must have domain-wide</span>
-<span class="sd"> delegation enabled.</span>
-<span class="sd"> :type delegate_to: string</span>
-<span class="sd"> """</span>
<span class="nb">super</span><span class="p">(</span><span class="n">DataProcPigOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
@@ -634,6 +710,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span> <span class="o">=</span> <span class="n">cluster_name</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dataproc_properties</span> <span class="o">=</span> <span class="n">dataproc_pig_properties</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span> <span class="o">=</span> <span class="n">dataproc_pig_jars</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">region</span> <span class="o">=</span> <span class="n">region</span>
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
@@ -649,14 +726,41 @@
<span class="n">job</span><span class="o">.</span><span class="n">add_jar_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span><span class="p">)</span>
<span class="n">job</span><span class="o">.</span><span class="n">set_job_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">)</span>
- <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">())</span></div>
+ <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div>
<div class="viewcode-block" id="DataProcHiveOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcHiveOperator">[docs]</a><span class="k">class</span> <span class="nc">DataProcHiveOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Start a Hive query Job on a Cloud DataProc cluster.</span>
+
+<span class="sd"> :param query: The query or reference to the query file (q extension).</span>
+<span class="sd"> :type query: string</span>
+<span class="sd"> :param query_uri: The uri of a hive script on Cloud Storage.</span>
+<span class="sd"> :type query_uri: string</span>
+<span class="sd"> :param variables: Map of named parameters for the query.</span>
+<span class="sd"> :type variables: dict</span>
+<span class="sd"> :param job_name: The job name used in the DataProc cluster. This name by default</span>
+<span class="sd"> is the task_id appended with the execution data, but can be templated. The</span>
+<span class="sd"> name will always be appended with a random number to avoid name clashes.</span>
+<span class="sd"> :type job_name: string</span>
+<span class="sd"> :param cluster_name: The name of the DataProc cluster.</span>
+<span class="sd"> :type cluster_name: string</span>
+<span class="sd"> :param dataproc_hive_properties: Map for the Pig properties. Ideal to put in</span>
+<span class="sd"> default arguments</span>
+<span class="sd"> :type dataproc_hive_properties: dict</span>
+<span class="sd"> :param dataproc_hive_jars: URIs to jars provisioned in Cloud Storage (example: for</span>
+<span class="sd"> UDFs and libs) and are ideal to put in default arguments.</span>
+<span class="sd"> :type dataproc_hive_jars: list</span>
+<span class="sd"> :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request must have domain-wide</span>
+<span class="sd"> delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> :param region: The specified region where the dataproc cluster is created.</span>
+<span class="sd"> :type region: string</span>
<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'query'</span><span class="p">,</span> <span class="s1">'variables'</span><span class="p">,</span> <span class="s1">'job_name'</span><span class="p">,</span> <span class="s1">'cluster_name'</span><span class="p">]</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'query'</span><span class="p">,</span> <span class="s1">'variables'</span><span class="p">,</span> <span class="s1">'job_name'</span><span class="p">,</span> <span class="s1">'cluster_name'</span><span class="p">,</span> <span class="s1">'dataproc_jars'</span><span class="p">]</span>
<span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.q'</span><span class="p">,)</span>
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#0273d4'</span>
@@ -672,36 +776,10 @@
<span class="n">dataproc_hive_jars</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">region</span><span class="o">=</span><span class="s1">'global'</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span>
<span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Create a new DataProcHiveOperator.</span>
-
-<span class="sd"> :param query: The query or reference to the query file (q extension).</span>
-<span class="sd"> :type query: string</span>
-<span class="sd"> :param query_uri: The uri of a hive script on Cloud Storage.</span>
-<span class="sd"> :type query_uri: string</span>
-<span class="sd"> :param variables: Map of named parameters for the query.</span>
-<span class="sd"> :type variables: dict</span>
-<span class="sd"> :param job_name: The job name used in the DataProc cluster. This name by default</span>
-<span class="sd"> is the task_id appended with the execution data, but can be templated. The</span>
-<span class="sd"> name will always be appended with a random number to avoid name clashes.</span>
-<span class="sd"> :type job_name: string</span>
-<span class="sd"> :param cluster_name: The name of the DataProc cluster.</span>
-<span class="sd"> :type cluster_name: string</span>
-<span class="sd"> :param dataproc_hive_properties: Map for the Pig properties. Ideal to put in</span>
-<span class="sd"> default arguments</span>
-<span class="sd"> :type dataproc_hive_properties: dict</span>
-<span class="sd"> :param dataproc_hive_jars: URIs to jars provisioned in Cloud Storage (example: for</span>
-<span class="sd"> UDFs and libs) and are ideal to put in default arguments.</span>
-<span class="sd"> :type dataproc_hive_jars: list</span>
-<span class="sd"> :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.</span>
-<span class="sd"> :type gcp_conn_id: string</span>
-<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
-<span class="sd"> For this to work, the service account making the request must have domain-wide</span>
-<span class="sd"> delegation enabled.</span>
-<span class="sd"> :type delegate_to: string</span>
-<span class="sd"> """</span>
+
<span class="nb">super</span><span class="p">(</span><span class="n">DataProcHiveOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
@@ -712,6 +790,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span> <span class="o">=</span> <span class="n">cluster_name</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dataproc_properties</span> <span class="o">=</span> <span class="n">dataproc_hive_properties</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span> <span class="o">=</span> <span class="n">dataproc_hive_jars</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">region</span> <span class="o">=</span> <span class="n">region</span>
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
@@ -728,14 +807,41 @@
<span class="n">job</span><span class="o">.</span><span class="n">add_jar_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span><span class="p">)</span>
<span class="n">job</span><span class="o">.</span><span class="n">set_job_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">)</span>
- <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">())</span></div>
+ <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div>
<div class="viewcode-block" id="DataProcSparkSqlOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcSparkSqlOperator">[docs]</a><span class="k">class</span> <span class="nc">DataProcSparkSqlOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Start a Spark SQL query Job on a Cloud DataProc cluster.</span>
+
+<span class="sd"> :param query: The query or reference to the query file (q extension).</span>
+<span class="sd"> :type query: string</span>
+<span class="sd"> :param query_uri: The uri of a spark sql script on Cloud Storage.</span>
+<span class="sd"> :type query_uri: string</span>
+<span class="sd"> :param variables: Map of named parameters for the query.</span>
+<span class="sd"> :type variables: dict</span>
+<span class="sd"> :param job_name: The job name used in the DataProc cluster. This name by default</span>
+<span class="sd"> is the task_id appended with the execution data, but can be templated. The</span>
+<span class="sd"> name will always be appended with a random number to avoid name clashes.</span>
+<span class="sd"> :type job_name: string</span>
+<span class="sd"> :param cluster_name: The name of the DataProc cluster.</span>
+<span class="sd"> :type cluster_name: string</span>
+<span class="sd"> :param dataproc_spark_properties: Map for the Pig properties. Ideal to put in</span>
+<span class="sd"> default arguments</span>
+<span class="sd"> :type dataproc_spark_properties: dict</span>
+<span class="sd"> :param dataproc_spark_jars: URIs to jars provisioned in Cloud Storage (example:</span>
+<span class="sd"> for UDFs and libs) and are ideal to put in default arguments.</span>
+<span class="sd"> :type dataproc_spark_jars: list</span>
+<span class="sd"> :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request must have domain-wide</span>
+<span class="sd"> delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> :param region: The specified region where the dataproc cluster is created.</span>
+<span class="sd"> :type region: string</span>
<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'query'</span><span class="p">,</span> <span class="s1">'variables'</span><span class="p">,</span> <span class="s1">'job_name'</span><span class="p">,</span> <span class="s1">'cluster_name'</span><span class="p">]</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'query'</span><span class="p">,</span> <span class="s1">'variables'</span><span class="p">,</span> <span class="s1">'job_name'</span><span class="p">,</span> <span class="s1">'cluster_name'</span><span class="p">,</span> <span class="s1">'dataproc_jars'</span><span class="p">]</span>
<span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.q'</span><span class="p">,)</span>
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#0273d4'</span>
@@ -751,36 +857,10 @@
<span class="n">dataproc_spark_jars</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">region</span><span class="o">=</span><span class="s1">'global'</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span>
<span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Create a new DataProcSparkSqlOperator.</span>
-
-<span class="sd"> :param query: The query or reference to the query file (q extension).</span>
-<span class="sd"> :type query: string</span>
-<span class="sd"> :param query_uri: The uri of a spark sql script on Cloud Storage.</span>
-<span class="sd"> :type query_uri: string</span>
-<span class="sd"> :param variables: Map of named parameters for the query.</span>
-<span class="sd"> :type variables: dict</span>
-<span class="sd"> :param job_name: The job name used in the DataProc cluster. This name by default</span>
-<span class="sd"> is the task_id appended with the execution data, but can be templated. The</span>
-<span class="sd"> name will always be appended with a random number to avoid name clashes.</span>
-<span class="sd"> :type job_name: string</span>
-<span class="sd"> :param cluster_name: The name of the DataProc cluster.</span>
-<span class="sd"> :type cluster_name: string</span>
-<span class="sd"> :param dataproc_spark_properties: Map for the Pig properties. Ideal to put in</span>
-<span class="sd"> default arguments</span>
-<span class="sd"> :type dataproc_spark_properties: dict</span>
-<span class="sd"> :param dataproc_spark_jars: URIs to jars provisioned in Cloud Storage (example:</span>
-<span class="sd"> for UDFs and libs) and are ideal to put in default arguments.</span>
-<span class="sd"> :type dataproc_spark_jars: list</span>
-<span class="sd"> :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.</span>
-<span class="sd"> :type gcp_conn_id: string</span>
-<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
-<span class="sd"> For this to work, the service account making the request must have domain-wide</span>
-<span class="sd"> delegation enabled.</span>
-<span class="sd"> :type delegate_to: string</span>
-<span class="sd"> """</span>
+
<span class="nb">super</span><span class="p">(</span><span class="n">DataProcSparkSqlOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
@@ -791,6 +871,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span> <span class="o">=</span> <span class="n">cluster_name</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dataproc_properties</span> <span class="o">=</span> <span class="n">dataproc_spark_properties</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span> <span class="o">=</span> <span class="n">dataproc_spark_jars</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">region</span> <span class="o">=</span> <span class="n">region</span>
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
@@ -807,15 +888,49 @@
<span class="n">job</span><span class="o">.</span><span class="n">add_jar_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span><span class="p">)</span>
<span class="n">job</span><span class="o">.</span><span class="n">set_job_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">)</span>
- <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">())</span></div>
+ <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div>
<div class="viewcode-block" id="DataProcSparkOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcSparkOperator">[docs]</a><span class="k">class</span> <span class="nc">DataProcSparkOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Start a Spark Job on a Cloud DataProc cluster.</span>
+
+<span class="sd"> :param main_jar: URI of the job jar provisioned on Cloud Storage. (use this or</span>
+<span class="sd"> the main_class, not both together).</span>
+<span class="sd"> :type main_jar: string</span>
+<span class="sd"> :param main_class: Name of the job class. (use this or the main_jar, not both</span>
+<span class="sd"> together).</span>
+<span class="sd"> :type main_class: string</span>
+<span class="sd"> :param arguments: Arguments for the job.</span>
+<span class="sd"> :type arguments: list</span>
+<span class="sd"> :param archives: List of archived files that will be unpacked in the work</span>
+<span class="sd"> directory. Should be stored in Cloud Storage.</span>
+<span class="sd"> :type archives: list</span>
+<span class="sd"> :param files: List of files to be copied to the working directory</span>
+<span class="sd"> :type files: list</span>
+<span class="sd"> :param job_name: The job name used in the DataProc cluster. This name by default</span>
+<span class="sd"> is the task_id appended with the execution data, but can be templated. The</span>
+<span class="sd"> name will always be appended with a random number to avoid name clashes.</span>
+<span class="sd"> :type job_name: string</span>
+<span class="sd"> :param cluster_name: The name of the DataProc cluster.</span>
+<span class="sd"> :type cluster_name: string</span>
+<span class="sd"> :param dataproc_spark_properties: Map for the Pig properties. Ideal to put in</span>
+<span class="sd"> default arguments</span>
+<span class="sd"> :type dataproc_spark_properties: dict</span>
+<span class="sd"> :param dataproc_spark_jars: URIs to jars provisioned in Cloud Storage (example:</span>
+<span class="sd"> for UDFs and libs) and are ideal to put in default arguments.</span>
+<span class="sd"> :type dataproc_spark_jars: list</span>
+<span class="sd"> :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request must have domain-wide</span>
+<span class="sd"> delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> :param region: The specified region where the dataproc cluster is created.</span>
+<span class="sd"> :type region: string</span>
<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'arguments'</span><span class="p">,</span> <span class="s1">'job_name'</span><span class="p">,</span> <span class="s1">'cluster_name'</span><span class="p">]</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'arguments'</span><span class="p">,</span> <span class="s1">'job_name'</span><span class="p">,</span> <span class="s1">'cluster_name'</span><span class="p">,</span> <span class="s1">'dataproc_jars'</span><span class="p">]</span>
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#0273d4'</span>
<span class="nd">@apply_defaults</span>
@@ -832,43 +947,10 @@
<span class="n">dataproc_spark_jars</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">region</span><span class="o">=</span><span class="s1">'global'</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span>
<span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Create a new DataProcSparkOperator.</span>
-<span class="sd"> :param main_jar: URI of the job jar provisioned on Cloud Storage. (use this or</span>
-<span class="sd"> the main_class, not both together).</span>
-<span class="sd"> :type main_jar: string</span>
-<span class="sd"> :param main_class: Name of the job class. (use this or the main_jar, not both</span>
-<span class="sd"> together).</span>
-<span class="sd"> :type main_class: string</span>
-<span class="sd"> :param arguments: Arguments for the job.</span>
-<span class="sd"> :type arguments: list</span>
-<span class="sd"> :param archives: List of archived files that will be unpacked in the work</span>
-<span class="sd"> directory. Should be stored in Cloud Storage.</span>
-<span class="sd"> :type archives: list</span>
-<span class="sd"> :param files: List of files to be copied to the working directory</span>
-<span class="sd"> :type files: list</span>
-<span class="sd"> :param job_name: The job name used in the DataProc cluster. This name by default</span>
-<span class="sd"> is the task_id appended with the execution data, but can be templated. The</span>
-<span class="sd"> name will always be appended with a random number to avoid name clashes.</span>
-<span class="sd"> :type job_name: string</span>
-<span class="sd"> :param cluster_name: The name of the DataProc cluster.</span>
-<span class="sd"> :type cluster_name: string</span>
-<span class="sd"> :param dataproc_spark_properties: Map for the Pig properties. Ideal to put in</span>
-<span class="sd"> default arguments</span>
-<span class="sd"> :type dataproc_spark_properties: dict</span>
-<span class="sd"> :param dataproc_spark_jars: URIs to jars provisioned in Cloud Storage (example:</span>
-<span class="sd"> for UDFs and libs) and are ideal to put in default arguments.</span>
-<span class="sd"> :type dataproc_spark_jars: list</span>
-<span class="sd"> :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.</span>
-<span class="sd"> :type gcp_conn_id: string</span>
-<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
-<span class="sd"> For this to work, the service account making the request must have domain-wide</span>
-<span class="sd"> delegation enabled.</span>
-<span class="sd"> :type delegate_to: string</span>
-<span class="sd"> """</span>
<span class="nb">super</span><span class="p">(</span><span class="n">DataProcSparkOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
@@ -881,6 +963,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span> <span class="o">=</span> <span class="n">cluster_name</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dataproc_properties</span> <span class="o">=</span> <span class="n">dataproc_spark_properties</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span> <span class="o">=</span> <span class="n">dataproc_spark_jars</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">region</span> <span class="o">=</span> <span class="n">region</span>
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
@@ -895,15 +978,49 @@
<span class="n">job</span><span class="o">.</span><span class="n">add_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">files</span><span class="p">)</span>
<span class="n">job</span><span class="o">.</span><span class="n">set_job_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">)</span>
- <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">())</span></div>
+ <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div>
<div class="viewcode-block" id="DataProcHadoopOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcHadoopOperator">[docs]</a><span class="k">class</span> <span class="nc">DataProcHadoopOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Start a Hadoop Job on a Cloud DataProc cluster.</span>
+
+<span class="sd"> :param main_jar: URI of the job jar provisioned on Cloud Storage. (use this or</span>
+<span class="sd"> the main_class, not both together).</span>
+<span class="sd"> :type main_jar: string</span>
+<span class="sd"> :param main_class: Name of the job class. (use this or the main_jar, not both</span>
+<span class="sd"> together).</span>
+<span class="sd"> :type main_class: string</span>
+<span class="sd"> :param arguments: Arguments for the job.</span>
+<span class="sd"> :type arguments: list</span>
+<span class="sd"> :param archives: List of archived files that will be unpacked in the work</span>
+<span class="sd"> directory. Should be stored in Cloud Storage.</span>
+<span class="sd"> :type archives: list</span>
+<span class="sd"> :param files: List of files to be copied to the working directory</span>
+<span class="sd"> :type files: list</span>
+<span class="sd"> :param job_name: The job name used in the DataProc cluster. This name by default</span>
+<span class="sd"> is the task_id appended with the execution data, but can be templated. The</span>
+<span class="sd"> name will always be appended with a random number to avoid name clashes.</span>
+<span class="sd"> :type job_name: string</span>
+<span class="sd"> :param cluster_name: The name of the DataProc cluster.</span>
+<span class="sd"> :type cluster_name: string</span>
+<span class="sd"> :param dataproc_hadoop_properties: Map for the Pig properties. Ideal to put in</span>
+<span class="sd"> default arguments</span>
+<span class="sd"> :type dataproc_hadoop_properties: dict</span>
+<span class="sd"> :param dataproc_hadoop_jars: URIs to jars provisioned in Cloud Storage (example:</span>
+<span class="sd"> for UDFs and libs) and are ideal to put in default arguments.</span>
+<span class="sd"> :type dataproc_hadoop_jars: list</span>
+<span class="sd"> :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request must have domain-wide</span>
+<span class="sd"> delegation enabled.</s
<TRUNCATED>
[22/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/sensors/aws_redshift_cluster_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/sensors/aws_redshift_cluster_sensor.html b/_modules/airflow/contrib/sensors/aws_redshift_cluster_sensor.html
new file mode 100644
index 0000000..16a47e3
--- /dev/null
+++ b/_modules/airflow/contrib/sensors/aws_redshift_cluster_sensor.html
@@ -0,0 +1,287 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.sensors.aws_redshift_cluster_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.sensors.aws_redshift_cluster_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.sensors.aws_redshift_cluster_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.redshift_hook</span> <span class="k">import</span> <span class="n">RedshiftHook</span>
+<span class="kn">from</span> <span class="nn">airflow.sensors.base_sensor_operator</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="AwsRedshiftClusterSensor"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.sensors.aws_redshift_cluster_sensor.AwsRedshiftClusterSensor">[docs]</a><span class="k">class</span> <span class="nc">AwsRedshiftClusterSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Waits for a Redshift cluster to reach a specific status.</span>
+
+<span class="sd"> :param cluster_identifier: The identifier for the cluster being pinged.</span>
+<span class="sd"> :type cluster_identifier: str</span>
+<span class="sd"> :param target_status: The cluster status desired.</span>
+<span class="sd"> :type target_status: str</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'cluster_identifier'</span><span class="p">,</span> <span class="s1">'target_status'</span><span class="p">)</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">cluster_identifier</span><span class="p">,</span>
+ <span class="n">target_status</span><span class="o">=</span><span class="s1">'available'</span><span class="p">,</span>
+ <span class="n">aws_conn_id</span><span class="o">=</span><span class="s1">'aws_default'</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">AwsRedshiftClusterSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cluster_identifier</span> <span class="o">=</span> <span class="n">cluster_identifier</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">target_status</span> <span class="o">=</span> <span class="n">target_status</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span> <span class="o">=</span> <span class="n">aws_conn_id</span>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Poking for status : </span><span class="si">{self.target_status}</span><span class="se">\n</span><span class="s1">'</span>
+ <span class="s1">'for cluster </span><span class="si">{self.cluster_identifier}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">RedshiftHook</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">cluster_status</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cluster_identifier</span><span class="p">)</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">target_status</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/sensors/bash_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/sensors/bash_sensor.html b/_modules/airflow/contrib/sensors/bash_sensor.html
new file mode 100644
index 0000000..f754fda
--- /dev/null
+++ b/_modules/airflow/contrib/sensors/bash_sensor.html
@@ -0,0 +1,331 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.sensors.bash_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.sensors.bash_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.sensors.bash_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">bytes</span>
+<span class="kn">import</span> <span class="nn">os</span>
+<span class="kn">from</span> <span class="nn">subprocess</span> <span class="k">import</span> <span class="n">Popen</span><span class="p">,</span> <span class="n">STDOUT</span><span class="p">,</span> <span class="n">PIPE</span>
+<span class="kn">from</span> <span class="nn">tempfile</span> <span class="k">import</span> <span class="n">gettempdir</span><span class="p">,</span> <span class="n">NamedTemporaryFile</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.sensors.base_sensor_operator</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.file</span> <span class="k">import</span> <span class="n">TemporaryDirectory</span>
+
+
+<div class="viewcode-block" id="BashSensor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.sensors.bash_sensor.BashSensor">[docs]</a><span class="k">class</span> <span class="nc">BashSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Executes a bash command/script and returns True if and only if the</span>
+<span class="sd"> return code is 0.</span>
+
+<span class="sd"> :param bash_command: The command, set of commands or reference to a</span>
+<span class="sd"> bash script (must be '.sh') to be executed.</span>
+<span class="sd"> :type bash_command: string</span>
+
+<span class="sd"> :param env: If env is not None, it must be a mapping that defines the</span>
+<span class="sd"> environment variables for the new process; these are used instead</span>
+<span class="sd"> of inheriting the current process environment, which is the default</span>
+<span class="sd"> behavior. (templated)</span>
+<span class="sd"> :type env: dict</span>
+<span class="sd"> :param output_encoding: output encoding of bash command.</span>
+<span class="sd"> :type output_encoding: string</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bash_command'</span><span class="p">,</span> <span class="s1">'env'</span><span class="p">)</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">bash_command</span><span class="p">,</span>
+ <span class="n">env</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">output_encoding</span><span class="o">=</span><span class="s1">'utf-8'</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">BashSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bash_command</span> <span class="o">=</span> <span class="n">bash_command</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">env</span> <span class="o">=</span> <span class="n">env</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">output_encoding</span> <span class="o">=</span> <span class="n">output_encoding</span>
+
+<div class="viewcode-block" id="BashSensor.poke"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.sensors.bash_sensor.BashSensor.poke">[docs]</a> <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Execute the bash command in a temporary directory</span>
+<span class="sd"> which will be cleaned afterwards</span>
+<span class="sd"> """</span>
+ <span class="n">bash_command</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">bash_command</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Tmp dir root location: </span><span class="se">\n</span><span class="s2"> </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">gettempdir</span><span class="p">())</span>
+ <span class="k">with</span> <span class="n">TemporaryDirectory</span><span class="p">(</span><span class="n">prefix</span><span class="o">=</span><span class="s1">'airflowtmp'</span><span class="p">)</span> <span class="k">as</span> <span class="n">tmp_dir</span><span class="p">:</span>
+ <span class="k">with</span> <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="nb">dir</span><span class="o">=</span><span class="n">tmp_dir</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
+
+ <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="nb">bytes</span><span class="p">(</span><span class="n">bash_command</span><span class="p">,</span> <span class="s1">'utf_8'</span><span class="p">))</span>
+ <span class="n">f</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
+ <span class="n">fname</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">name</span>
+ <span class="n">script_location</span> <span class="o">=</span> <span class="n">tmp_dir</span> <span class="o">+</span> <span class="s2">"/"</span> <span class="o">+</span> <span class="n">fname</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s2">"Temporary script location: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
+ <span class="n">script_location</span>
+ <span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Running command: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">bash_command</span><span class="p">)</span>
+ <span class="n">sp</span> <span class="o">=</span> <span class="n">Popen</span><span class="p">(</span>
+ <span class="p">[</span><span class="s1">'bash'</span><span class="p">,</span> <span class="n">fname</span><span class="p">],</span>
+ <span class="n">stdout</span><span class="o">=</span><span class="n">PIPE</span><span class="p">,</span> <span class="n">stderr</span><span class="o">=</span><span class="n">STDOUT</span><span class="p">,</span>
+ <span class="n">close_fds</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">cwd</span><span class="o">=</span><span class="n">tmp_dir</span><span class="p">,</span>
+ <span class="n">env</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">env</span><span class="p">,</span> <span class="n">preexec_fn</span><span class="o">=</span><span class="n">os</span><span class="o">.</span><span class="n">setsid</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">sp</span> <span class="o">=</span> <span class="n">sp</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Output:"</span><span class="p">)</span>
+ <span class="n">line</span> <span class="o">=</span> <span class="s1">''</span>
+ <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="nb">iter</span><span class="p">(</span><span class="n">sp</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">readline</span><span class="p">,</span> <span class="sa">b</span><span class="s1">''</span><span class="p">):</span>
+ <span class="n">line</span> <span class="o">=</span> <span class="n">line</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">output_encoding</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">line</span><span class="p">)</span>
+ <span class="n">sp</span><span class="o">.</span><span class="n">wait</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Command exited with "</span>
+ <span class="s2">"return code </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">sp</span><span class="o">.</span><span class="n">returncode</span><span class="p">))</span>
+
+ <span class="k">return</span> <span class="ow">not</span> <span class="n">sp</span><span class="o">.</span><span class="n">returncode</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/sensors/bigquery_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/sensors/bigquery_sensor.html b/_modules/airflow/contrib/sensors/bigquery_sensor.html
new file mode 100644
index 0000000..871562f
--- /dev/null
+++ b/_modules/airflow/contrib/sensors/bigquery_sensor.html
@@ -0,0 +1,306 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.sensors.bigquery_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.sensors.bigquery_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.sensors.bigquery_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">from</span> <span class="nn">airflow.sensors.base_sensor_operator</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="k">import</span> <span class="n">BigQueryHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="BigQueryTableSensor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.sensors.bigquery_sensor.BigQueryTableSensor">[docs]</a><span class="k">class</span> <span class="nc">BigQueryTableSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Checks for the existence of a table in Google Bigquery.</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'dataset_id'</span><span class="p">,</span> <span class="s1">'table_id'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f0eee4'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span>
+ <span class="n">dataset_id</span><span class="p">,</span>
+ <span class="n">table_id</span><span class="p">,</span>
+ <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">'bigquery_default_conn'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Create a new BigQueryTableSensor.</span>
+
+<span class="sd"> :param project_id: The Google cloud project in which to look for the table. The connection supplied to the hook</span>
+<span class="sd"> must provide access to the specified project.</span>
+<span class="sd"> :type project_id: string</span>
+<span class="sd"> :param dataset_id: The name of the dataset in which to look for the table.</span>
+<span class="sd"> storage bucket.</span>
+<span class="sd"> :type dataset_id: string</span>
+<span class="sd"> :param table_id: The name of the table to check the existence of.</span>
+<span class="sd"> :type table_id: string</span>
+<span class="sd"> :param bigquery_conn_id: The connection ID to use when connecting to Google BigQuery.</span>
+<span class="sd"> :type bigquery_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request must have domain-wide delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> """</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryTableSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">dataset_id</span> <span class="o">=</span> <span class="n">dataset_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">table_id</span> <span class="o">=</span> <span class="n">table_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">table_uri</span> <span class="o">=</span> <span class="s1">'</span><span class="si">{0}</span><span class="s1">:</span><span class="si">{1}</span><span class="s1">.</span><span class="si">{2}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Sensor checks existence of table: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">table_uri</span><span class="p">)</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span>
+ <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">table_exists</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_id</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/sensors/datadog_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/sensors/datadog_sensor.html b/_modules/airflow/contrib/sensors/datadog_sensor.html
new file mode 100644
index 0000000..15fb7b8
--- /dev/null
+++ b/_modules/airflow/contrib/sensors/datadog_sensor.html
@@ -0,0 +1,318 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.sensors.datadog_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.sensors.datadog_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.sensors.datadog_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">from</span> <span class="nn">airflow.sensors.base_sensor_operator</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.datadog_hook</span> <span class="k">import</span> <span class="n">DatadogHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">datadog</span> <span class="k">import</span> <span class="n">api</span>
+
+
+<div class="viewcode-block" id="DatadogSensor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.sensors.datadog_sensor.DatadogSensor">[docs]</a><span class="k">class</span> <span class="nc">DatadogSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> A sensor to listen, with a filter, to datadog event streams and determine</span>
+<span class="sd"> if some event was emitted.</span>
+
+<span class="sd"> Depends on the datadog API, which has to be deployed on the same server where</span>
+<span class="sd"> Airflow runs.</span>
+
+<span class="sd"> :param datadog_conn_id: The connection to datadog, containing metadata for api keys.</span>
+<span class="sd"> :param datadog_conn_id: string</span>
+<span class="sd"> """</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#66c3dd'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">datadog_conn_id</span><span class="o">=</span><span class="s1">'datadog_default'</span><span class="p">,</span>
+ <span class="n">from_seconds_ago</span><span class="o">=</span><span class="mi">3600</span><span class="p">,</span>
+ <span class="n">up_to_seconds_from_now</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
+ <span class="n">priority</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">sources</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">tags</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">response_check</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">DatadogSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">datadog_conn_id</span> <span class="o">=</span> <span class="n">datadog_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">from_seconds_ago</span> <span class="o">=</span> <span class="n">from_seconds_ago</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">up_to_seconds_from_now</span> <span class="o">=</span> <span class="n">up_to_seconds_from_now</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">priority</span> <span class="o">=</span> <span class="n">priority</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sources</span> <span class="o">=</span> <span class="n">sources</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">tags</span> <span class="o">=</span> <span class="n">tags</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">response_check</span> <span class="o">=</span> <span class="n">response_check</span>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="c1"># This instantiates the hook, but doesn't need it further,</span>
+ <span class="c1"># because the API authenticates globally (unfortunately),</span>
+ <span class="c1"># but for airflow this shouldn't matter too much, because each</span>
+ <span class="c1"># task instance runs in its own process anyway.</span>
+ <span class="n">DatadogHook</span><span class="p">(</span><span class="n">datadog_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">datadog_conn_id</span><span class="p">)</span>
+
+ <span class="n">response</span> <span class="o">=</span> <span class="n">api</span><span class="o">.</span><span class="n">Event</span><span class="o">.</span><span class="n">query</span><span class="p">(</span>
+ <span class="n">start</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">from_seconds_ago</span><span class="p">,</span>
+ <span class="n">end</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">up_to_seconds_from_now</span><span class="p">,</span>
+ <span class="n">priority</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">priority</span><span class="p">,</span>
+ <span class="n">sources</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">sources</span><span class="p">,</span>
+ <span class="n">tags</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">tags</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">response</span><span class="p">,</span> <span class="nb">dict</span><span class="p">)</span> <span class="ow">and</span> <span class="n">response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'status'</span><span class="p">,</span> <span class="s1">'ok'</span><span class="p">)</span> <span class="o">!=</span> <span class="s1">'ok'</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Unexpected Datadog result: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">response</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Datadog returned unexpected result"</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">response_check</span><span class="p">:</span>
+ <span class="c1"># run content check on response</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">response_check</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
+
+ <span class="c1"># If no check was inserted, assume any event that matched yields true.</span>
+ <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="n">response</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/sensors/emr_base_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/sensors/emr_base_sensor.html b/_modules/airflow/contrib/sensors/emr_base_sensor.html
new file mode 100644
index 0000000..dead0c5
--- /dev/null
+++ b/_modules/airflow/contrib/sensors/emr_base_sensor.html
@@ -0,0 +1,291 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.sensors.emr_base_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.sensors.emr_base_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.sensors.emr_base_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">from</span> <span class="nn">airflow.sensors.base_sensor_operator</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+
+
+<div class="viewcode-block" id="EmrBaseSensor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.sensors.emr_base_sensor.EmrBaseSensor">[docs]</a><span class="k">class</span> <span class="nc">EmrBaseSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Contains general sensor behavior for EMR.</span>
+<span class="sd"> Subclasses should implement get_emr_response() and state_from_response() methods.</span>
+<span class="sd"> Subclasses should also implement NON_TERMINAL_STATES and FAILED_STATE constants.</span>
+<span class="sd"> """</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#66c3ff'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">aws_conn_id</span><span class="o">=</span><span class="s1">'aws_default'</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">EmrBaseSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span> <span class="o">=</span> <span class="n">aws_conn_id</span>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_emr_response</span><span class="p">()</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">response</span><span class="p">[</span><span class="s1">'ResponseMetadata'</span><span class="p">][</span><span class="s1">'HTTPStatusCode'</span><span class="p">]</span> <span class="o">==</span> <span class="mi">200</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Bad HTTP response: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">response</span><span class="p">)</span>
+ <span class="k">return</span> <span class="kc">False</span>
+
+ <span class="n">state</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">state_from_response</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Job flow currently </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">state</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">state</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">NON_TERMINAL_STATES</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">False</span>
+
+ <span class="k">if</span> <span class="n">state</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">FAILED_STATE</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'EMR job failed'</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="kc">True</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[45/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/gcp_dataproc_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/gcp_dataproc_hook.html b/_modules/airflow/contrib/hooks/gcp_dataproc_hook.html
new file mode 100644
index 0000000..4ca7edd
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/gcp_dataproc_hook.html
@@ -0,0 +1,463 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.gcp_dataproc_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.gcp_dataproc_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.gcp_dataproc_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+<span class="kn">import</span> <span class="nn">time</span>
+<span class="kn">import</span> <span class="nn">uuid</span>
+
+<span class="kn">from</span> <span class="nn">apiclient.discovery</span> <span class="k">import</span> <span class="n">build</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="k">import</span> <span class="n">GoogleCloudBaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+
+<span class="k">class</span> <span class="nc">_DataProcJob</span><span class="p">(</span><span class="n">LoggingMixin</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataproc_api</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="p">,</span> <span class="n">region</span><span class="o">=</span><span class="s1">'global'</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_api</span> <span class="o">=</span> <span class="n">dataproc_api</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">region</span> <span class="o">=</span> <span class="n">region</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job</span> <span class="o">=</span> <span class="n">dataproc_api</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">regions</span><span class="p">()</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">region</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">,</span>
+ <span class="n">body</span><span class="o">=</span><span class="n">job</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">[</span><span class="s1">'reference'</span><span class="p">][</span><span class="s1">'jobId'</span><span class="p">]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'DataProc job </span><span class="si">%s</span><span class="s1"> is </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">[</span><span class="s1">'status'</span><span class="p">][</span><span class="s1">'state'</span><span class="p">])</span>
+ <span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">wait_for_done</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_api</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">regions</span><span class="p">()</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">region</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">,</span>
+ <span class="n">jobId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">job_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">num_retries</span><span class="o">=</span><span class="mi">5</span><span class="p">)</span>
+ <span class="k">if</span> <span class="s1">'ERROR'</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">[</span><span class="s1">'status'</span><span class="p">][</span><span class="s1">'state'</span><span class="p">]:</span>
+ <span class="nb">print</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">))</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">'DataProc job </span><span class="si">%s</span><span class="s1"> has errors'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">[</span><span class="s1">'status'</span><span class="p">][</span><span class="s1">'details'</span><span class="p">])</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">))</span>
+ <span class="k">return</span> <span class="kc">False</span>
+ <span class="k">if</span> <span class="s1">'CANCELLED'</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">[</span><span class="s1">'status'</span><span class="p">][</span><span class="s1">'state'</span><span class="p">]:</span>
+ <span class="nb">print</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">))</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s1">'DataProc job </span><span class="si">%s</span><span class="s1"> is cancelled'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span><span class="p">)</span>
+ <span class="k">if</span> <span class="s1">'details'</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">[</span><span class="s1">'status'</span><span class="p">]:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">[</span><span class="s1">'status'</span><span class="p">][</span><span class="s1">'details'</span><span class="p">])</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">))</span>
+ <span class="k">return</span> <span class="kc">False</span>
+ <span class="k">if</span> <span class="s1">'DONE'</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">[</span><span class="s1">'status'</span><span class="p">][</span><span class="s1">'state'</span><span class="p">]:</span>
+ <span class="k">return</span> <span class="kc">True</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
+ <span class="s1">'DataProc job </span><span class="si">%s</span><span class="s1"> is </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">[</span><span class="s1">'status'</span><span class="p">][</span><span class="s1">'state'</span><span class="p">])</span>
+ <span class="p">)</span>
+ <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">raise_error</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">message</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="k">if</span> <span class="s1">'ERROR'</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">[</span><span class="s1">'status'</span><span class="p">][</span><span class="s1">'state'</span><span class="p">]:</span>
+ <span class="k">if</span> <span class="n">message</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">message</span> <span class="o">=</span> <span class="s2">"Google DataProc job has error"</span>
+ <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="n">message</span> <span class="o">+</span> <span class="s2">": "</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">[</span><span class="s1">'status'</span><span class="p">][</span><span class="s1">'details'</span><span class="p">]))</span>
+
+ <span class="k">def</span> <span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">job</span>
+
+
+<span class="k">class</span> <span class="nc">_DataProcJobBuilder</span><span class="p">:</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">cluster_name</span><span class="p">,</span> <span class="n">job_type</span><span class="p">,</span> <span class="n">properties</span><span class="p">):</span>
+ <span class="n">name</span> <span class="o">=</span> <span class="n">task_id</span> <span class="o">+</span> <span class="s2">"_"</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">uuid</span><span class="o">.</span><span class="n">uuid1</span><span class="p">())[:</span><span class="mi">8</span><span class="p">]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job_type</span> <span class="o">=</span> <span class="n">job_type</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s2">"job"</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s2">"reference"</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s2">"projectId"</span><span class="p">:</span> <span class="n">project_id</span><span class="p">,</span>
+ <span class="s2">"jobId"</span><span class="p">:</span> <span class="n">name</span><span class="p">,</span>
+ <span class="p">},</span>
+ <span class="s2">"placement"</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s2">"clusterName"</span><span class="p">:</span> <span class="n">cluster_name</span>
+ <span class="p">},</span>
+ <span class="n">job_type</span><span class="p">:</span> <span class="p">{</span>
+ <span class="p">}</span>
+ <span class="p">}</span>
+ <span class="p">}</span>
+ <span class="k">if</span> <span class="n">properties</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">[</span><span class="s2">"job"</span><span class="p">][</span><span class="n">job_type</span><span class="p">][</span><span class="s2">"properties"</span><span class="p">]</span> <span class="o">=</span> <span class="n">properties</span>
+
+ <span class="k">def</span> <span class="nf">add_variables</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">variables</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">variables</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">[</span><span class="s2">"job"</span><span class="p">][</span><span class="bp">self</span><span class="o">.</span><span class="n">job_type</span><span class="p">][</span><span class="s2">"scriptVariables"</span><span class="p">]</span> <span class="o">=</span> <span class="n">variables</span>
+
+ <span class="k">def</span> <span class="nf">add_args</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">args</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">args</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">[</span><span class="s2">"job"</span><span class="p">][</span><span class="bp">self</span><span class="o">.</span><span class="n">job_type</span><span class="p">][</span><span class="s2">"args"</span><span class="p">]</span> <span class="o">=</span> <span class="n">args</span>
+
+ <span class="k">def</span> <span class="nf">add_query</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">[</span><span class="s2">"job"</span><span class="p">][</span><span class="bp">self</span><span class="o">.</span><span class="n">job_type</span><span class="p">][</span><span class="s2">"queryList"</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'queries'</span><span class="p">:</span> <span class="p">[</span><span class="n">query</span><span class="p">]}</span>
+
+ <span class="k">def</span> <span class="nf">add_query_uri</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query_uri</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">[</span><span class="s2">"job"</span><span class="p">][</span><span class="bp">self</span><span class="o">.</span><span class="n">job_type</span><span class="p">][</span><span class="s2">"queryFileUri"</span><span class="p">]</span> <span class="o">=</span> <span class="n">query_uri</span>
+
+ <span class="k">def</span> <span class="nf">add_jar_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">jars</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">jars</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">[</span><span class="s2">"job"</span><span class="p">][</span><span class="bp">self</span><span class="o">.</span><span class="n">job_type</span><span class="p">][</span><span class="s2">"jarFileUris"</span><span class="p">]</span> <span class="o">=</span> <span class="n">jars</span>
+
+ <span class="k">def</span> <span class="nf">add_archive_uris</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">archives</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">archives</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">[</span><span class="s2">"job"</span><span class="p">][</span><span class="bp">self</span><span class="o">.</span><span class="n">job_type</span><span class="p">][</span><span class="s2">"archiveUris"</span><span class="p">]</span> <span class="o">=</span> <span class="n">archives</span>
+
+ <span class="k">def</span> <span class="nf">add_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">files</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">files</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">[</span><span class="s2">"job"</span><span class="p">][</span><span class="bp">self</span><span class="o">.</span><span class="n">job_type</span><span class="p">][</span><span class="s2">"fileUris"</span><span class="p">]</span> <span class="o">=</span> <span class="n">files</span>
+
+ <span class="k">def</span> <span class="nf">add_python_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">pyfiles</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">pyfiles</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">[</span><span class="s2">"job"</span><span class="p">][</span><span class="bp">self</span><span class="o">.</span><span class="n">job_type</span><span class="p">][</span><span class="s2">"pythonFileUris"</span><span class="p">]</span> <span class="o">=</span> <span class="n">pyfiles</span>
+
+ <span class="k">def</span> <span class="nf">set_main</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">main_jar</span><span class="p">,</span> <span class="n">main_class</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">main_class</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">main_jar</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s2">"Set either main_jar or main_class"</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">main_jar</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">[</span><span class="s2">"job"</span><span class="p">][</span><span class="bp">self</span><span class="o">.</span><span class="n">job_type</span><span class="p">][</span><span class="s2">"mainJarFileUri"</span><span class="p">]</span> <span class="o">=</span> <span class="n">main_jar</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">[</span><span class="s2">"job"</span><span class="p">][</span><span class="bp">self</span><span class="o">.</span><span class="n">job_type</span><span class="p">][</span><span class="s2">"mainClass"</span><span class="p">]</span> <span class="o">=</span> <span class="n">main_class</span>
+
+ <span class="k">def</span> <span class="nf">set_python_main</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">main</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">[</span><span class="s2">"job"</span><span class="p">][</span><span class="bp">self</span><span class="o">.</span><span class="n">job_type</span><span class="p">][</span><span class="s2">"mainPythonFileUri"</span><span class="p">]</span> <span class="o">=</span> <span class="n">main</span>
+
+ <span class="k">def</span> <span class="nf">set_job_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job</span><span class="p">[</span><span class="s2">"job"</span><span class="p">][</span><span class="s2">"reference"</span><span class="p">][</span><span class="s2">"jobId"</span><span class="p">]</span> <span class="o">=</span> <span class="n">name</span> <span class="o">+</span> <span class="s2">"_"</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">uuid</span><span class="o">.</span><span class="n">uuid1</span><span class="p">())[:</span><span class="mi">8</span><span class="p">]</span>
+
+ <span class="k">def</span> <span class="nf">build</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">job</span>
+
+
+<span class="k">class</span> <span class="nc">_DataProcOperation</span><span class="p">(</span><span class="n">LoggingMixin</span><span class="p">):</span>
+ <span class="sd">"""Continuously polls Dataproc Operation until it completes."""</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataproc_api</span><span class="p">,</span> <span class="n">operation</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_api</span> <span class="o">=</span> <span class="n">dataproc_api</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">operation</span> <span class="o">=</span> <span class="n">operation</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">operation_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">operation</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]</span>
+
+ <span class="k">def</span> <span class="nf">wait_for_done</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_check_done</span><span class="p">():</span>
+ <span class="k">return</span> <span class="kc">True</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'Waiting for Dataproc Operation </span><span class="si">%s</span><span class="s1"> to finish'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">operation_name</span><span class="p">)</span>
+ <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+ <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">10</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">operation</span> <span class="o">=</span> <span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_api</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span>
+ <span class="o">.</span><span class="n">regions</span><span class="p">()</span>
+ <span class="o">.</span><span class="n">operations</span><span class="p">()</span>
+ <span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">operation_name</span><span class="p">)</span>
+ <span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">num_retries</span><span class="o">=</span><span class="mi">5</span><span class="p">))</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_check_done</span><span class="p">():</span>
+ <span class="k">return</span> <span class="kc">True</span>
+
+ <span class="k">def</span> <span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">operation</span>
+
+ <span class="k">def</span> <span class="nf">_check_done</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="s1">'done'</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">operation</span><span class="p">:</span>
+ <span class="k">if</span> <span class="s1">'error'</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">operation</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
+ <span class="s1">'Dataproc Operation </span><span class="si">%s</span><span class="s1"> failed with error: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">operation_name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">operation</span><span class="p">[</span><span class="s1">'error'</span><span class="p">][</span><span class="s1">'message'</span><span class="p">])</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_raise_error</span><span class="p">()</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'Dataproc Operation </span><span class="si">%s</span><span class="s1"> done'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">operation</span><span class="p">[</span><span class="s1">'name'</span><span class="p">])</span>
+ <span class="k">return</span> <span class="kc">True</span>
+ <span class="k">return</span> <span class="kc">False</span>
+
+ <span class="k">def</span> <span class="nf">_raise_error</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s1">'Google Dataproc Operation </span><span class="si">%s</span><span class="s1"> failed: </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span>
+ <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">operation_name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">operation</span><span class="p">[</span><span class="s1">'error'</span><span class="p">][</span><span class="s1">'message'</span><span class="p">]))</span>
+
+
+<div class="viewcode-block" id="DataProcHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.gcp_dataproc_hook.DataProcHook">[docs]</a><span class="k">class</span> <span class="nc">DataProcHook</span><span class="p">(</span><span class="n">GoogleCloudBaseHook</span><span class="p">):</span>
+ <span class="sd">"""Hook for Google Cloud Dataproc APIs."""</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="o">=</span><span class="s1">'v1beta2'</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">DataProcHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">api_version</span> <span class="o">=</span> <span class="n">api_version</span>
+
+<div class="viewcode-block" id="DataProcHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.gcp_dataproc_hook.DataProcHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""Returns a Google Cloud Dataproc service object."""</span>
+ <span class="n">http_authorized</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">build</span><span class="p">(</span><span class="s1">'dataproc'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">api_version</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">)</span></div>
+
+ <span class="k">def</span> <span class="nf">get_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">region</span><span class="p">,</span> <span class="n">cluster_name</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">regions</span><span class="p">()</span><span class="o">.</span><span class="n">clusters</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">region</span><span class="o">=</span><span class="n">region</span><span class="p">,</span>
+ <span class="n">clusterName</span><span class="o">=</span><span class="n">cluster_name</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">num_retries</span><span class="o">=</span><span class="mi">5</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">submit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="p">,</span> <span class="n">region</span><span class="o">=</span><span class="s1">'global'</span><span class="p">):</span>
+ <span class="n">submitted</span> <span class="o">=</span> <span class="n">_DataProcJob</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">(),</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="p">,</span> <span class="n">region</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">submitted</span><span class="o">.</span><span class="n">wait_for_done</span><span class="p">():</span>
+ <span class="n">submitted</span><span class="o">.</span><span class="n">raise_error</span><span class="p">(</span><span class="s1">'DataProcTask has errors'</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">create_job_template</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">cluster_name</span><span class="p">,</span> <span class="n">job_type</span><span class="p">,</span> <span class="n">properties</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">_DataProcJobBuilder</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">cluster_name</span><span class="p">,</span>
+ <span class="n">job_type</span><span class="p">,</span> <span class="n">properties</span><span class="p">)</span>
+
+<div class="viewcode-block" id="DataProcHook.await"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.gcp_dataproc_hook.DataProcHook.await">[docs]</a> <span class="k">def</span> <span class="nf">await</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">operation</span><span class="p">):</span>
+ <span class="sd">"""Awaits for Google Cloud Dataproc Operation to complete."""</span>
+ <span class="n">submitted</span> <span class="o">=</span> <span class="n">_DataProcOperation</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">(),</span> <span class="n">operation</span><span class="p">)</span>
+ <span class="n">submitted</span><span class="o">.</span><span class="n">wait_for_done</span><span class="p">()</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/gcp_mlengine_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/gcp_mlengine_hook.html b/_modules/airflow/contrib/hooks/gcp_mlengine_hook.html
index 82450ae..df9ea25 100644
--- a/_modules/airflow/contrib/hooks/gcp_mlengine_hook.html
+++ b/_modules/airflow/contrib/hooks/gcp_mlengine_hook.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -236,14 +238,14 @@
<span class="sd"> :type project_id: string</span>
<span class="sd"> :param job: MLEngine Job object that should be provided to the MLEngine</span>
-<span class="sd"> API, such as:</span>
-<span class="sd"> {</span>
-<span class="sd"> 'jobId': 'my_job_id',</span>
-<span class="sd"> 'trainingInput': {</span>
-<span class="sd"> 'scaleTier': 'STANDARD_1',</span>
-<span class="sd"> ...</span>
-<span class="sd"> }</span>
-<span class="sd"> }</span>
+<span class="sd"> API, such as: ::</span>
+<span class="sd"> {</span>
+<span class="sd"> 'jobId': 'my_job_id',</span>
+<span class="sd"> 'trainingInput': {</span>
+<span class="sd"> 'scaleTier': 'STANDARD_1',</span>
+<span class="sd"> ...</span>
+<span class="sd"> }</span>
+<span class="sd"> }</span>
<span class="sd"> :type job: dict</span>
<span class="sd"> :param use_existing_job_fn: In case that a MLEngine job with the same</span>
[34/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/databricks_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/databricks_operator.html b/_modules/airflow/contrib/operators/databricks_operator.html
index cb38349..c39f769 100644
--- a/_modules/airflow/contrib/operators/databricks_operator.html
+++ b/_modules/airflow/contrib/operators/databricks_operator.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,22 @@
<h1>Source code for airflow.contrib.operators.databricks_operator</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="c1">#</span>
<span class="kn">import</span> <span class="nn">six</span>
@@ -190,6 +197,10 @@
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="n">XCOM_RUN_ID_KEY</span> <span class="o">=</span> <span class="s1">'run_id'</span>
+<span class="n">XCOM_RUN_PAGE_URL_KEY</span> <span class="o">=</span> <span class="s1">'run_page_url'</span>
+
+
<div class="viewcode-block" id="DatabricksSubmitRunOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator">[docs]</a><span class="k">class</span> <span class="nc">DatabricksSubmitRunOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Submits an Spark job run to Databricks using the</span>
@@ -307,6 +318,8 @@
<span class="sd"> :param databricks_retry_limit: Amount of times retry if the Databricks backend is</span>
<span class="sd"> unreachable. Its value must be greater than or equal to 1.</span>
<span class="sd"> :type databricks_retry_limit: int</span>
+<span class="sd"> :param do_xcom_push: Whether we should push run_id and run_page_url to xcom.</span>
+<span class="sd"> :type do_xcom_push: boolean</span>
<span class="sd"> """</span>
<span class="c1"># Used in airflow.models.BaseOperator</span>
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'json'</span><span class="p">,)</span>
@@ -327,6 +340,7 @@
<span class="n">databricks_conn_id</span><span class="o">=</span><span class="s1">'databricks_default'</span><span class="p">,</span>
<span class="n">polling_period_seconds</span><span class="o">=</span><span class="mi">30</span><span class="p">,</span>
<span class="n">databricks_retry_limit</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span>
+ <span class="n">do_xcom_push</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Creates a new ``DatabricksSubmitRunOperator``.</span>
@@ -356,6 +370,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">json</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_deep_string_coerce</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">json</span><span class="p">)</span>
<span class="c1"># This variable will be used in case our task gets killed.</span>
<span class="bp">self</span><span class="o">.</span><span class="n">run_id</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">do_xcom_push</span> <span class="o">=</span> <span class="n">do_xcom_push</span>
<span class="k">def</span> <span class="nf">_deep_string_coerce</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">content</span><span class="p">,</span> <span class="n">json_path</span><span class="o">=</span><span class="s1">'json'</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -393,8 +408,12 @@
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">hook</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">run_id</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">submit_run</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">json</span><span class="p">)</span>
- <span class="n">run_page_url</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_run_page_url</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">run_id</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">do_xcom_push</span><span class="p">:</span>
+ <span class="n">context</span><span class="p">[</span><span class="s1">'ti'</span><span class="p">]</span><span class="o">.</span><span class="n">xcom_push</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">XCOM_RUN_ID_KEY</span><span class="p">,</span> <span class="n">value</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">run_id</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Run submitted with run_id: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_id</span><span class="p">)</span>
+ <span class="n">run_page_url</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_run_page_url</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">run_id</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">do_xcom_push</span><span class="p">:</span>
+ <span class="n">context</span><span class="p">[</span><span class="s1">'ti'</span><span class="p">]</span><span class="o">.</span><span class="n">xcom_push</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">XCOM_RUN_PAGE_URL_KEY</span><span class="p">,</span> <span class="n">value</span><span class="o">=</span><span class="n">run_page_url</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_log_run_page_url</span><span class="p">(</span><span class="n">run_page_url</span><span class="p">)</span>
<span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
<span class="n">run_state</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_run_state</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">run_id</span><span class="p">)</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/dataflow_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/dataflow_operator.html b/_modules/airflow/contrib/operators/dataflow_operator.html
index 6807e24..d5d4041 100644
--- a/_modules/airflow/contrib/operators/dataflow_operator.html
+++ b/_modules/airflow/contrib/operators/dataflow_operator.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -169,25 +171,31 @@
<h1>Source code for airflow.contrib.operators.dataflow_operator</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
-<span class="kn">import</span> <span class="nn">copy</span>
<span class="kn">import</span> <span class="nn">re</span>
<span class="kn">import</span> <span class="nn">uuid</span>
+<span class="kn">import</span> <span class="nn">copy</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcs_hook</span> <span class="k">import</span> <span class="n">GoogleCloudStorageHook</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_dataflow_hook</span> <span class="k">import</span> <span class="n">DataFlowHook</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.version</span> <span class="k">import</span> <span class="n">version</span>
<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
@@ -199,32 +207,35 @@
<span class="sd"> It's a good practice to define dataflow_* parameters in the default_args of the dag</span>
<span class="sd"> like the project, zone and staging location.</span>
-<span class="sd"> ```</span>
-<span class="sd"> default_args = {</span>
-<span class="sd"> 'dataflow_default_options': {</span>
-<span class="sd"> 'project': 'my-gcp-project',</span>
-<span class="sd"> 'zone': 'europe-west1-d',</span>
-<span class="sd"> 'stagingLocation': 'gs://my-staging-bucket/staging/'</span>
-<span class="sd"> }</span>
-<span class="sd"> }</span>
-<span class="sd"> ```</span>
+<span class="sd"> .. code-block:: python</span>
+
+<span class="sd"> default_args = {</span>
+<span class="sd"> 'dataflow_default_options': {</span>
+<span class="sd"> 'project': 'my-gcp-project',</span>
+<span class="sd"> 'zone': 'europe-west1-d',</span>
+<span class="sd"> 'stagingLocation': 'gs://my-staging-bucket/staging/'</span>
+<span class="sd"> }</span>
+<span class="sd"> }</span>
<span class="sd"> You need to pass the path to your dataflow as a file reference with the ``jar``</span>
-<span class="sd"> parameter, the jar needs to be a self executing jar. Use ``options`` to pass on</span>
-<span class="sd"> options to your job.</span>
-
-<span class="sd"> ```</span>
-<span class="sd"> t1 = DataFlowOperation(</span>
-<span class="sd"> task_id='datapflow_example',</span>
-<span class="sd"> jar='{{var.value.gcp_dataflow_base}}pipeline/build/libs/pipeline-example-1.0.jar',</span>
-<span class="sd"> options={</span>
-<span class="sd"> 'autoscalingAlgorithm': 'BASIC',</span>
-<span class="sd"> 'maxNumWorkers': '50',</span>
-<span class="sd"> 'start': '{{ds}}',</span>
-<span class="sd"> 'partitionType': 'DAY'</span>
-<span class="sd"> },</span>
-<span class="sd"> dag=my-dag)</span>
-<span class="sd"> ```</span>
+<span class="sd"> parameter, the jar needs to be a self executing jar (see documentation here:</span>
+<span class="sd"> https://beam.apache.org/documentation/runners/dataflow/#self-executing-jar).</span>
+<span class="sd"> Use ``options`` to pass on options to your job.</span>
+
+<span class="sd"> .. code-block:: python</span>
+
+<span class="sd"> t1 = DataFlowOperation(</span>
+<span class="sd"> task_id='datapflow_example',</span>
+<span class="sd"> jar='{{var.value.gcp_dataflow_base}}pipeline/build/libs/pipeline-example-1.0.jar',</span>
+<span class="sd"> options={</span>
+<span class="sd"> 'autoscalingAlgorithm': 'BASIC',</span>
+<span class="sd"> 'maxNumWorkers': '50',</span>
+<span class="sd"> 'start': '{{ds}}',</span>
+<span class="sd"> 'partitionType': 'DAY',</span>
+<span class="sd"> 'labels': {'foo' : 'bar'}</span>
+<span class="sd"> },</span>
+<span class="sd"> gcp_conn_id='gcp-airflow-service-account',</span>
+<span class="sd"> dag=my-dag)</span>
<span class="sd"> Both ``jar`` and ``options`` are templated so you can use variables in them.</span>
<span class="sd"> """</span>
@@ -239,6 +250,8 @@
<span class="n">options</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">poll_sleep</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span>
+ <span class="n">job_class</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span>
<span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -248,9 +261,10 @@
<span class="sd"> high-level options, for instances, project and zone information, which</span>
<span class="sd"> apply to all dataflow operators in the DAG.</span>
-<span class="sd"> For more detail on job submission have a look at the reference:</span>
-<span class="sd"> https://cloud.google.com/dataflow/pipelines/specifying-exec-params</span>
+<span class="sd"> .. seealso::</span>
+<span class="sd"> For more detail on job submission have a look at the reference:</span>
+<span class="sd"> https://cloud.google.com/dataflow/pipelines/specifying-exec-params</span>
<span class="sd"> :param jar: The reference to a self executing DataFlow jar.</span>
<span class="sd"> :type jar: string</span>
@@ -265,29 +279,147 @@
<span class="sd"> For this to work, the service account making the request must have</span>
<span class="sd"> domain-wide delegation enabled.</span>
<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> :param poll_sleep: The time in seconds to sleep between polling Google</span>
+<span class="sd"> Cloud Platform for the dataflow job status while the job is in the</span>
+<span class="sd"> JOB_STATE_RUNNING state.</span>
+<span class="sd"> :type poll_sleep: int</span>
+<span class="sd"> :param job_class: The name of the dataflow job class to be executued, it</span>
+<span class="sd"> is often not the main class configured in the dataflow jar file.</span>
+<span class="sd"> :type job_class: string</span>
<span class="sd"> """</span>
<span class="nb">super</span><span class="p">(</span><span class="n">DataFlowJavaOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="n">dataflow_default_options</span> <span class="o">=</span> <span class="n">dataflow_default_options</span> <span class="ow">or</span> <span class="p">{}</span>
<span class="n">options</span> <span class="o">=</span> <span class="n">options</span> <span class="ow">or</span> <span class="p">{}</span>
-
+ <span class="n">options</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="s1">'labels'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
+ <span class="p">{</span><span class="s1">'airflow-version'</span><span class="p">:</span> <span class="s1">'v'</span> <span class="o">+</span> <span class="n">version</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'.'</span><span class="p">,</span> <span class="s1">'-'</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'+'</span><span class="p">,</span> <span class="s1">'-'</span><span class="p">)})</span>
<span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
<span class="bp">self</span><span class="o">.</span><span class="n">jar</span> <span class="o">=</span> <span class="n">jar</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dataflow_default_options</span> <span class="o">=</span> <span class="n">dataflow_default_options</span>
<span class="bp">self</span><span class="o">.</span><span class="n">options</span> <span class="o">=</span> <span class="n">options</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">poll_sleep</span> <span class="o">=</span> <span class="n">poll_sleep</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job_class</span> <span class="o">=</span> <span class="n">job_class</span>
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">bucket_helper</span> <span class="o">=</span> <span class="n">GoogleCloudBucketHelper</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">jar</span> <span class="o">=</span> <span class="n">bucket_helper</span><span class="o">.</span><span class="n">google_cloud_to_local</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">jar</span><span class="p">)</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">DataFlowHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
- <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">,</span>
+ <span class="n">poll_sleep</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">poll_sleep</span><span class="p">)</span>
<span class="n">dataflow_options</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataflow_default_options</span><span class="p">)</span>
<span class="n">dataflow_options</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">options</span><span class="p">)</span>
- <span class="n">hook</span><span class="o">.</span><span class="n">start_java_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="n">dataflow_options</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">jar</span><span class="p">)</span></div>
+ <span class="n">hook</span><span class="o">.</span><span class="n">start_java_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="n">dataflow_options</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">jar</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_class</span><span class="p">)</span></div>
+
+
+<div class="viewcode-block" id="DataflowTemplateOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataflow_operator.DataflowTemplateOperator">[docs]</a><span class="k">class</span> <span class="nc">DataflowTemplateOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Start a Templated Cloud DataFlow batch job. The parameters of the operation</span>
+<span class="sd"> will be passed to the job.</span>
+<span class="sd"> It's a good practice to define dataflow_* parameters in the default_args of the dag</span>
+<span class="sd"> like the project, zone and staging location.</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> https://cloud.google.com/dataflow/docs/reference/rest/v1b3/LaunchTemplateParameters</span>
+<span class="sd"> https://cloud.google.com/dataflow/docs/reference/rest/v1b3/RuntimeEnvironment</span>
+
+<span class="sd"> .. code-block:: python</span>
+
+<span class="sd"> default_args = {</span>
+<span class="sd"> 'dataflow_default_options': {</span>
+<span class="sd"> 'project': 'my-gcp-project'</span>
+<span class="sd"> 'zone': 'europe-west1-d',</span>
+<span class="sd"> 'tempLocation': 'gs://my-staging-bucket/staging/'</span>
+<span class="sd"> }</span>
+<span class="sd"> }</span>
+<span class="sd"> }</span>
+
+<span class="sd"> You need to pass the path to your dataflow template as a file reference with the</span>
+<span class="sd"> ``template`` parameter. Use ``parameters`` to pass on parameters to your job.</span>
+<span class="sd"> Use ``environment`` to pass on runtime environment variables to your job.</span>
+
+<span class="sd"> .. code-block:: python</span>
+
+<span class="sd"> t1 = DataflowTemplateOperator(</span>
+<span class="sd"> task_id='datapflow_example',</span>
+<span class="sd"> template='{{var.value.gcp_dataflow_base}}',</span>
+<span class="sd"> parameters={</span>
+<span class="sd"> 'inputFile': "gs://bucket/input/my_input.txt",</span>
+<span class="sd"> 'outputFile': "gs://bucket/output/my_output.txt"</span>
+<span class="sd"> },</span>
+<span class="sd"> gcp_conn_id='gcp-airflow-service-account',</span>
+<span class="sd"> dag=my-dag)</span>
+
+<span class="sd"> ``template``, ``dataflow_default_options`` and ``parameters`` are templated so you can</span>
+<span class="sd"> use variables in them.</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'parameters'</span><span class="p">,</span> <span class="s1">'dataflow_default_options'</span><span class="p">,</span> <span class="s1">'template'</span><span class="p">]</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#0273d4'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">template</span><span class="p">,</span>
+ <span class="n">dataflow_default_options</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">poll_sleep</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Create a new DataflowTemplateOperator. Note that</span>
+<span class="sd"> dataflow_default_options is expected to save high-level options</span>
+<span class="sd"> for project information, which apply to all dataflow operators in the DAG.</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> https://cloud.google.com/dataflow/docs/reference/rest/v1b3</span>
+<span class="sd"> /LaunchTemplateParameters</span>
+<span class="sd"> https://cloud.google.com/dataflow/docs/reference/rest/v1b3/RuntimeEnvironment</span>
+<span class="sd"> For more detail on job template execution have a look at the reference:</span>
+<span class="sd"> https://cloud.google.com/dataflow/docs/templates/executing-templates</span>
+
+<span class="sd"> :param template: The reference to the DataFlow template.</span>
+<span class="sd"> :type template: string</span>
+<span class="sd"> :param dataflow_default_options: Map of default job environment options.</span>
+<span class="sd"> :type dataflow_default_options: dict</span>
+<span class="sd"> :param parameters: Map of job specific parameters for the template.</span>
+<span class="sd"> :type parameters: dict</span>
+<span class="sd"> :param gcp_conn_id: The connection ID to use connecting to Google Cloud</span>
+<span class="sd"> Platform.</span>
+<span class="sd"> :type gcp_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request must have</span>
+<span class="sd"> domain-wide delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> :param poll_sleep: The time in seconds to sleep between polling Google</span>
+<span class="sd"> Cloud Platform for the dataflow job status while the job is in the</span>
+<span class="sd"> JOB_STATE_RUNNING state.</span>
+<span class="sd"> :type poll_sleep: int</span>
+<span class="sd"> """</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">DataflowTemplateOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="n">dataflow_default_options</span> <span class="o">=</span> <span class="n">dataflow_default_options</span> <span class="ow">or</span> <span class="p">{}</span>
+ <span class="n">parameters</span> <span class="o">=</span> <span class="n">parameters</span> <span class="ow">or</span> <span class="p">{}</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">dataflow_default_options</span> <span class="o">=</span> <span class="n">dataflow_default_options</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">poll_sleep</span> <span class="o">=</span> <span class="n">poll_sleep</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">template</span> <span class="o">=</span> <span class="n">template</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span> <span class="o">=</span> <span class="n">parameters</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">DataFlowHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">,</span>
+ <span class="n">poll_sleep</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">poll_sleep</span><span class="p">)</span>
+
+ <span class="n">hook</span><span class="o">.</span><span class="n">start_template_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataflow_default_options</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">template</span><span class="p">)</span></div>
<div class="viewcode-block" id="DataFlowPythonOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataflow_operator.DataFlowPythonOperator">[docs]</a><span class="k">class</span> <span class="nc">DataFlowPythonOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
@@ -303,6 +435,7 @@
<span class="n">options</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">poll_sleep</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span>
<span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -312,12 +445,12 @@
<span class="sd"> high-level options, for instances, project and zone information, which</span>
<span class="sd"> apply to all dataflow operators in the DAG.</span>
-<span class="sd"> For more detail on job submission have a look at the reference:</span>
+<span class="sd"> .. seealso::</span>
+<span class="sd"> For more detail on job submission have a look at the reference:</span>
+<span class="sd"> https://cloud.google.com/dataflow/pipelines/specifying-exec-params</span>
-<span class="sd"> https://cloud.google.com/dataflow/pipelines/specifying-exec-params</span>
-
-<span class="sd"> :param py_file: Reference to the python dataflow pipleline file, e.g.,</span>
-<span class="sd"> /some/local/file/path/to/your/python/pipeline/file.py.</span>
+<span class="sd"> :param py_file: Reference to the python dataflow pipleline file.py, e.g.,</span>
+<span class="sd"> /some/local/file/path/to/your/python/pipeline/file.</span>
<span class="sd"> :type py_file: string</span>
<span class="sd"> :param py_options: Additional python options.</span>
<span class="sd"> :type pyt_options: list of strings, e.g., ["-m", "-v"].</span>
@@ -332,6 +465,10 @@
<span class="sd"> For this to work, the service account making the request must have</span>
<span class="sd"> domain-wide delegation enabled.</span>
<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> :param poll_sleep: The time in seconds to sleep between polling Google</span>
+<span class="sd"> Cloud Platform for the dataflow job status while the job is in the</span>
+<span class="sd"> JOB_STATE_RUNNING state.</span>
+<span class="sd"> :type poll_sleep: int</span>
<span class="sd"> """</span>
<span class="nb">super</span><span class="p">(</span><span class="n">DataFlowPythonOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
@@ -339,16 +476,20 @@
<span class="bp">self</span><span class="o">.</span><span class="n">py_options</span> <span class="o">=</span> <span class="n">py_options</span> <span class="ow">or</span> <span class="p">[]</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dataflow_default_options</span> <span class="o">=</span> <span class="n">dataflow_default_options</span> <span class="ow">or</span> <span class="p">{}</span>
<span class="bp">self</span><span class="o">.</span><span class="n">options</span> <span class="o">=</span> <span class="n">options</span> <span class="ow">or</span> <span class="p">{}</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">options</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="s1">'labels'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
+ <span class="p">{</span><span class="s1">'airflow-version'</span><span class="p">:</span> <span class="s1">'v'</span> <span class="o">+</span> <span class="n">version</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'.'</span><span class="p">,</span> <span class="s1">'-'</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'+'</span><span class="p">,</span> <span class="s1">'-'</span><span class="p">)})</span>
<span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">poll_sleep</span> <span class="o">=</span> <span class="n">poll_sleep</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="DataFlowPythonOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataflow_operator.DataFlowPythonOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="sd">"""Execute the python dataflow job."""</span>
<span class="n">bucket_helper</span> <span class="o">=</span> <span class="n">GoogleCloudBucketHelper</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">py_file</span> <span class="o">=</span> <span class="n">bucket_helper</span><span class="o">.</span><span class="n">google_cloud_to_local</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">py_file</span><span class="p">)</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">DataFlowHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
- <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">,</span>
+ <span class="n">poll_sleep</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">poll_sleep</span><span class="p">)</span>
<span class="n">dataflow_options</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataflow_default_options</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">dataflow_options</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">options</span><span class="p">)</span>
<span class="c1"># Convert argument names from lowerCamelCase to snake case.</span>
@@ -358,7 +499,7 @@
<span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">dataflow_options</span><span class="p">}</span>
<span class="n">hook</span><span class="o">.</span><span class="n">start_python_dataflow</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="n">formatted_options</span><span class="p">,</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">py_file</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">py_options</span><span class="p">)</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="n">py_file</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">py_options</span><span class="p">)</span></div></div>
<span class="k">class</span> <span class="nc">GoogleCloudBucketHelper</span><span class="p">():</span>
[17/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/hooks/S3_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/hooks/S3_hook.html b/_modules/airflow/hooks/S3_hook.html
index f1c0617..8aa4ecd 100644
--- a/_modules/airflow/hooks/S3_hook.html
+++ b/_modules/airflow/hooks/S3_hook.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,22 @@
<h1>Source code for airflow.hooks.S3_hook</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.aws_hook</span> <span class="k">import</span> <span class="n">AwsHook</span>
@@ -201,13 +208,13 @@
<span class="k">def</span> <span class="nf">parse_s3_url</span><span class="p">(</span><span class="n">s3url</span><span class="p">):</span>
<span class="n">parsed_url</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">s3url</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span><span class="p">:</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Please provide a bucket_name'</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Please provide a bucket_name instead of "</span><span class="si">%s</span><span class="s1">"'</span> <span class="o">%</span> <span class="n">s3url</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">bucket_name</span> <span class="o">=</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span>
<span class="n">key</span> <span class="o">=</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">'/'</span><span class="p">)</span>
<span class="k">return</span> <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span>
- <span class="k">def</span> <span class="nf">check_for_bucket</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">):</span>
+<div class="viewcode-block" id="S3Hook.check_for_bucket"><a class="viewcode-back" href="../../../integration.html#airflow.hooks.S3_hook.S3Hook.check_for_bucket">[docs]</a> <span class="k">def</span> <span class="nf">check_for_bucket</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Check if bucket_name exists.</span>
@@ -218,9 +225,9 @@
<span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">head_bucket</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">True</span>
<span class="k">except</span><span class="p">:</span>
- <span class="k">return</span> <span class="kc">False</span>
+ <span class="k">return</span> <span class="kc">False</span></div>
- <span class="k">def</span> <span class="nf">get_bucket</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">):</span>
+<div class="viewcode-block" id="S3Hook.get_bucket"><a class="viewcode-back" href="../../../integration.html#airflow.hooks.S3_hook.S3Hook.get_bucket">[docs]</a> <span class="k">def</span> <span class="nf">get_bucket</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Returns a boto3.S3.Bucket object</span>
@@ -228,9 +235,9 @@
<span class="sd"> :type bucket_name: str</span>
<span class="sd"> """</span>
<span class="n">s3</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_resource_type</span><span class="p">(</span><span class="s1">'s3'</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">s3</span><span class="o">.</span><span class="n">Bucket</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">s3</span><span class="o">.</span><span class="n">Bucket</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">)</span></div>
- <span class="k">def</span> <span class="nf">check_for_prefix</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="p">,</span> <span class="n">delimiter</span><span class="p">):</span>
+<div class="viewcode-block" id="S3Hook.check_for_prefix"><a class="viewcode-back" href="../../../integration.html#airflow.hooks.S3_hook.S3Hook.check_for_prefix">[docs]</a> <span class="k">def</span> <span class="nf">check_for_prefix</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="p">,</span> <span class="n">delimiter</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Checks that a prefix exists in a bucket</span>
<span class="sd"> """</span>
@@ -238,9 +245,9 @@
<span class="n">prefix_split</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="sa">r</span><span class="s1">'(\w+[</span><span class="si">{d}</span><span class="s1">])$'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">d</span><span class="o">=</span><span class="n">delimiter</span><span class="p">),</span> <span class="n">prefix</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
<span class="n">previous_level</span> <span class="o">=</span> <span class="n">prefix_split</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">plist</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">list_prefixes</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">previous_level</span><span class="p">,</span> <span class="n">delimiter</span><span class="p">)</span>
- <span class="k">return</span> <span class="kc">False</span> <span class="k">if</span> <span class="n">plist</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">prefix</span> <span class="ow">in</span> <span class="n">plist</span>
+ <span class="k">return</span> <span class="kc">False</span> <span class="k">if</span> <span class="n">plist</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">prefix</span> <span class="ow">in</span> <span class="n">plist</span></div>
- <span class="k">def</span> <span class="nf">list_prefixes</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">''</span><span class="p">):</span>
+<div class="viewcode-block" id="S3Hook.list_prefixes"><a class="viewcode-back" href="../../../integration.html#airflow.hooks.S3_hook.S3Hook.list_prefixes">[docs]</a> <span class="k">def</span> <span class="nf">list_prefixes</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">''</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Lists prefixes in a bucket under prefix</span>
@@ -251,12 +258,23 @@
<span class="sd"> :param delimiter: the delimiter marks key hierarchy.</span>
<span class="sd"> :type delimiter: str</span>
<span class="sd"> """</span>
- <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">list_objects_v2</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span>
- <span class="n">Prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">,</span>
- <span class="n">Delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span>
- <span class="k">return</span> <span class="p">[</span><span class="n">p</span><span class="p">[</span><span class="s1">'Prefix'</span><span class="p">]</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">response</span><span class="p">[</span><span class="s1">'CommonPrefixes'</span><span class="p">]]</span> <span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'CommonPrefixes'</span><span class="p">)</span> <span class="k">else</span> <span class="kc">None</span>
-
- <span class="k">def</span> <span class="nf">list_keys</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">''</span><span class="p">):</span>
+ <span class="n">paginator</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">get_paginator</span><span class="p">(</span><span class="s1">'list_objects_v2'</span><span class="p">)</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="n">paginator</span><span class="o">.</span><span class="n">paginate</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span>
+ <span class="n">Prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">,</span>
+ <span class="n">Delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span>
+
+ <span class="n">has_results</span> <span class="o">=</span> <span class="kc">False</span>
+ <span class="n">prefixes</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="k">for</span> <span class="n">page</span> <span class="ow">in</span> <span class="n">response</span><span class="p">:</span>
+ <span class="k">if</span> <span class="s1">'CommonPrefixes'</span> <span class="ow">in</span> <span class="n">page</span><span class="p">:</span>
+ <span class="n">has_results</span> <span class="o">=</span> <span class="kc">True</span>
+ <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">page</span><span class="p">[</span><span class="s1">'CommonPrefixes'</span><span class="p">]:</span>
+ <span class="n">prefixes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">p</span><span class="p">[</span><span class="s1">'Prefix'</span><span class="p">])</span>
+
+ <span class="k">if</span> <span class="n">has_results</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">prefixes</span></div>
+
+<div class="viewcode-block" id="S3Hook.list_keys"><a class="viewcode-back" href="../../../integration.html#airflow.hooks.S3_hook.S3Hook.list_keys">[docs]</a> <span class="k">def</span> <span class="nf">list_keys</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">''</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Lists keys in a bucket under prefix and not containing delimiter</span>
@@ -267,12 +285,23 @@
<span class="sd"> :param delimiter: the delimiter marks key hierarchy.</span>
<span class="sd"> :type delimiter: str</span>
<span class="sd"> """</span>
- <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">list_objects_v2</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span>
- <span class="n">Prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">,</span>
- <span class="n">Delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span>
- <span class="k">return</span> <span class="p">[</span><span class="n">k</span><span class="p">[</span><span class="s1">'Key'</span><span class="p">]</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">response</span><span class="p">[</span><span class="s1">'Contents'</span><span class="p">]]</span> <span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Contents'</span><span class="p">)</span> <span class="k">else</span> <span class="kc">None</span>
-
- <span class="k">def</span> <span class="nf">check_for_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="n">paginator</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">get_paginator</span><span class="p">(</span><span class="s1">'list_objects_v2'</span><span class="p">)</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="n">paginator</span><span class="o">.</span><span class="n">paginate</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span>
+ <span class="n">Prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">,</span>
+ <span class="n">Delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span>
+
+ <span class="n">has_results</span> <span class="o">=</span> <span class="kc">False</span>
+ <span class="n">keys</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="k">for</span> <span class="n">page</span> <span class="ow">in</span> <span class="n">response</span><span class="p">:</span>
+ <span class="k">if</span> <span class="s1">'Contents'</span> <span class="ow">in</span> <span class="n">page</span><span class="p">:</span>
+ <span class="n">has_results</span> <span class="o">=</span> <span class="kc">True</span>
+ <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">page</span><span class="p">[</span><span class="s1">'Contents'</span><span class="p">]:</span>
+ <span class="n">keys</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">k</span><span class="p">[</span><span class="s1">'Key'</span><span class="p">])</span>
+
+ <span class="k">if</span> <span class="n">has_results</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">keys</span></div>
+
+<div class="viewcode-block" id="S3Hook.check_for_key"><a class="viewcode-back" href="../../../integration.html#airflow.hooks.S3_hook.S3Hook.check_for_key">[docs]</a> <span class="k">def</span> <span class="nf">check_for_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Checks if a key exists in a bucket</span>
@@ -288,9 +317,9 @@
<span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">head_object</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">Key</span><span class="o">=</span><span class="n">key</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">True</span>
<span class="k">except</span><span class="p">:</span>
- <span class="k">return</span> <span class="kc">False</span>
+ <span class="k">return</span> <span class="kc">False</span></div>
- <span class="k">def</span> <span class="nf">get_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+<div class="viewcode-block" id="S3Hook.get_key"><a class="viewcode-back" href="../../../integration.html#airflow.hooks.S3_hook.S3Hook.get_key">[docs]</a> <span class="k">def</span> <span class="nf">get_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Returns a boto3.s3.Object</span>
@@ -304,9 +333,9 @@
<span class="n">obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_resource_type</span><span class="p">(</span><span class="s1">'s3'</span><span class="p">)</span><span class="o">.</span><span class="n">Object</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span>
<span class="n">obj</span><span class="o">.</span><span class="n">load</span><span class="p">()</span>
- <span class="k">return</span> <span class="n">obj</span>
+ <span class="k">return</span> <span class="n">obj</span></div>
- <span class="k">def</span> <span class="nf">read_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+<div class="viewcode-block" id="S3Hook.read_key"><a class="viewcode-back" href="../../../integration.html#airflow.hooks.S3_hook.S3Hook.read_key">[docs]</a> <span class="k">def</span> <span class="nf">read_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Reads a key from S3</span>
@@ -317,23 +346,65 @@
<span class="sd"> """</span>
<span class="n">obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_key</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">obj</span><span class="o">.</span><span class="n">get</span><span class="p">()[</span><span class="s1">'Body'</span><span class="p">]</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">obj</span><span class="o">.</span><span class="n">get</span><span class="p">()[</span><span class="s1">'Body'</span><span class="p">]</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="S3Hook.select_key"><a class="viewcode-back" href="../../../integration.html#airflow.hooks.S3_hook.S3Hook.select_key">[docs]</a> <span class="k">def</span> <span class="nf">select_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">expression</span><span class="o">=</span><span class="s1">'SELECT * FROM S3Object'</span><span class="p">,</span>
+ <span class="n">expression_type</span><span class="o">=</span><span class="s1">'SQL'</span><span class="p">,</span>
+ <span class="n">input_serialization</span><span class="o">=</span><span class="p">{</span><span class="s1">'CSV'</span><span class="p">:</span> <span class="p">{}},</span>
+ <span class="n">output_serialization</span><span class="o">=</span><span class="p">{</span><span class="s1">'CSV'</span><span class="p">:</span> <span class="p">{}}):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Reads a key with S3 Select.</span>
+
+<span class="sd"> :param key: S3 key that will point to the file</span>
+<span class="sd"> :type key: str</span>
+<span class="sd"> :param bucket_name: Name of the bucket in which the file is stored</span>
+<span class="sd"> :type bucket_name: str</span>
+<span class="sd"> :param expression: S3 Select expression</span>
+<span class="sd"> :type expression: str</span>
+<span class="sd"> :param expression_type: S3 Select expression type</span>
+<span class="sd"> :type expression_type: str</span>
+<span class="sd"> :param input_serialization: S3 Select input data serialization format</span>
+<span class="sd"> :type input_serialization: dict</span>
+<span class="sd"> :param output_serialization: S3 Select output data serialization format</span>
+<span class="sd"> :type output_serialization: dict</span>
+<span class="sd"> :return: retrieved subset of original data by S3 Select</span>
+<span class="sd"> :rtype: str</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> For more details about S3 Select parameters:</span>
+<span class="sd"> http://boto3.readthedocs.io/en/latest/reference/services/s3.html#S3.Client.select_object_content</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">bucket_name</span><span class="p">:</span>
+ <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
+
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">select_object_content</span><span class="p">(</span>
+ <span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span>
+ <span class="n">Key</span><span class="o">=</span><span class="n">key</span><span class="p">,</span>
+ <span class="n">Expression</span><span class="o">=</span><span class="n">expression</span><span class="p">,</span>
+ <span class="n">ExpressionType</span><span class="o">=</span><span class="n">expression_type</span><span class="p">,</span>
+ <span class="n">InputSerialization</span><span class="o">=</span><span class="n">input_serialization</span><span class="p">,</span>
+ <span class="n">OutputSerialization</span><span class="o">=</span><span class="n">output_serialization</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="s1">''</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">event</span><span class="p">[</span><span class="s1">'Records'</span><span class="p">][</span><span class="s1">'Payload'</span><span class="p">]</span>
+ <span class="k">for</span> <span class="n">event</span> <span class="ow">in</span> <span class="n">response</span><span class="p">[</span><span class="s1">'Payload'</span><span class="p">]</span>
+ <span class="k">if</span> <span class="s1">'Records'</span> <span class="ow">in</span> <span class="n">event</span><span class="p">)</span></div>
- <span class="k">def</span> <span class="nf">check_for_wildcard_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+<div class="viewcode-block" id="S3Hook.check_for_wildcard_key"><a class="viewcode-back" href="../../../integration.html#airflow.hooks.S3_hook.S3Hook.check_for_wildcard_key">[docs]</a> <span class="k">def</span> <span class="nf">check_for_wildcard_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
<span class="n">wildcard_key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">''</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Checks that a key matching a wildcard expression exists in a bucket</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_wildcard_key</span><span class="p">(</span><span class="n">wildcard_key</span><span class="o">=</span><span class="n">wildcard_key</span><span class="p">,</span>
<span class="n">bucket_name</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span>
- <span class="n">delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+ <span class="n">delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span></div>
- <span class="k">def</span> <span class="nf">get_wildcard_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">wildcard_key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">''</span><span class="p">):</span>
+<div class="viewcode-block" id="S3Hook.get_wildcard_key"><a class="viewcode-back" href="../../../integration.html#airflow.hooks.S3_hook.S3Hook.get_wildcard_key">[docs]</a> <span class="k">def</span> <span class="nf">get_wildcard_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">wildcard_key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">''</span><span class="p">):</span>
<span class="sd">"""</span>
-<span class="sd"> Returns a boto3.s3.Object object matching the regular expression</span>
+<span class="sd"> Returns a boto3.s3.Object object matching the wildcard expression</span>
-<span class="sd"> :param regex_key: the path to the key</span>
-<span class="sd"> :type regex_key: str</span>
+<span class="sd"> :param wildcard_key: the path to the key</span>
+<span class="sd"> :type wildcard_key: str</span>
<span class="sd"> :param bucket_name: the name of the bucket</span>
<span class="sd"> :type bucket_name: str</span>
<span class="sd"> """</span>
@@ -345,9 +416,9 @@
<span class="k">if</span> <span class="n">klist</span><span class="p">:</span>
<span class="n">key_matches</span> <span class="o">=</span> <span class="p">[</span><span class="n">k</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">klist</span> <span class="k">if</span> <span class="n">fnmatch</span><span class="o">.</span><span class="n">fnmatch</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="n">wildcard_key</span><span class="p">)]</span>
<span class="k">if</span> <span class="n">key_matches</span><span class="p">:</span>
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_key</span><span class="p">(</span><span class="n">key_matches</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">bucket_name</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_key</span><span class="p">(</span><span class="n">key_matches</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">bucket_name</span><span class="p">)</span></div>
- <span class="k">def</span> <span class="nf">load_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+<div class="viewcode-block" id="S3Hook.load_file"><a class="viewcode-back" href="../../../integration.html#airflow.hooks.S3_hook.S3Hook.load_file">[docs]</a> <span class="k">def</span> <span class="nf">load_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
<span class="n">filename</span><span class="p">,</span>
<span class="n">key</span><span class="p">,</span>
<span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -381,9 +452,9 @@
<span class="n">extra_args</span><span class="p">[</span><span class="s1">'ServerSideEncryption'</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"AES256"</span>
<span class="n">client</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
- <span class="n">client</span><span class="o">.</span><span class="n">upload_file</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">ExtraArgs</span><span class="o">=</span><span class="n">extra_args</span><span class="p">)</span>
+ <span class="n">client</span><span class="o">.</span><span class="n">upload_file</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">ExtraArgs</span><span class="o">=</span><span class="n">extra_args</span><span class="p">)</span></div>
- <span class="k">def</span> <span class="nf">load_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+<div class="viewcode-block" id="S3Hook.load_string"><a class="viewcode-back" href="../../../integration.html#airflow.hooks.S3_hook.S3Hook.load_string">[docs]</a> <span class="k">def</span> <span class="nf">load_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
<span class="n">string_data</span><span class="p">,</span>
<span class="n">key</span><span class="p">,</span>
<span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -409,6 +480,37 @@
<span class="sd"> by S3 and will be stored in an encrypted form while at rest in S3.</span>
<span class="sd"> :type encrypt: bool</span>
<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">load_bytes</span><span class="p">(</span><span class="n">string_data</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">encoding</span><span class="p">),</span>
+ <span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">,</span>
+ <span class="n">bucket_name</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span>
+ <span class="n">replace</span><span class="o">=</span><span class="n">replace</span><span class="p">,</span>
+ <span class="n">encrypt</span><span class="o">=</span><span class="n">encrypt</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="S3Hook.load_bytes"><a class="viewcode-back" href="../../../integration.html#airflow.hooks.S3_hook.S3Hook.load_bytes">[docs]</a> <span class="k">def</span> <span class="nf">load_bytes</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">bytes_data</span><span class="p">,</span>
+ <span class="n">key</span><span class="p">,</span>
+ <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">replace</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">encrypt</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Loads bytes to S3</span>
+
+<span class="sd"> This is provided as a convenience to drop a string in S3. It uses the</span>
+<span class="sd"> boto infrastructure to ship a file to s3.</span>
+
+<span class="sd"> :param bytes_data: bytes to set as content for the key.</span>
+<span class="sd"> :type bytes_data: bytes</span>
+<span class="sd"> :param key: S3 key that will point to the file</span>
+<span class="sd"> :type key: str</span>
+<span class="sd"> :param bucket_name: Name of the bucket in which to store the file</span>
+<span class="sd"> :type bucket_name: str</span>
+<span class="sd"> :param replace: A flag to decide whether or not to overwrite the key</span>
+<span class="sd"> if it already exists</span>
+<span class="sd"> :type replace: bool</span>
+<span class="sd"> :param encrypt: If True, the file will be encrypted on the server-side</span>
+<span class="sd"> by S3 and will be stored in an encrypted form while at rest in S3.</span>
+<span class="sd"> :type encrypt: bool</span>
+<span class="sd"> """</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">bucket_name</span><span class="p">:</span>
<span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
@@ -419,10 +521,10 @@
<span class="k">if</span> <span class="n">encrypt</span><span class="p">:</span>
<span class="n">extra_args</span><span class="p">[</span><span class="s1">'ServerSideEncryption'</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"AES256"</span>
- <span class="n">filelike_buffer</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">(</span><span class="n">string_data</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">encoding</span><span class="p">))</span>
+ <span class="n">filelike_buffer</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">(</span><span class="n">bytes_data</span><span class="p">)</span>
<span class="n">client</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
- <span class="n">client</span><span class="o">.</span><span class="n">upload_fileobj</span><span class="p">(</span><span class="n">filelike_buffer</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">ExtraArgs</span><span class="o">=</span><span class="n">extra_args</span><span class="p">)</span></div>
+ <span class="n">client</span><span class="o">.</span><span class="n">upload_fileobj</span><span class="p">(</span><span class="n">filelike_buffer</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">ExtraArgs</span><span class="o">=</span><span class="n">extra_args</span><span class="p">)</span></div></div>
</pre></div>
</div>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/hooks/dbapi_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/hooks/dbapi_hook.html b/_modules/airflow/hooks/dbapi_hook.html
new file mode 100644
index 0000000..b8b8403
--- /dev/null
+++ b/_modules/airflow/hooks/dbapi_hook.html
@@ -0,0 +1,515 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.hooks.dbapi_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.hooks.dbapi_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.hooks.dbapi_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
+<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="k">import</span> <span class="n">basestring</span>
+<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span>
+<span class="kn">from</span> <span class="nn">contextlib</span> <span class="k">import</span> <span class="n">closing</span>
+<span class="kn">import</span> <span class="nn">sys</span>
+
+<span class="kn">from</span> <span class="nn">sqlalchemy</span> <span class="k">import</span> <span class="n">create_engine</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+
+
+<div class="viewcode-block" id="DbApiHook"><a class="viewcode-back" href="../../../code.html#airflow.hooks.dbapi_hook.DbApiHook">[docs]</a><span class="k">class</span> <span class="nc">DbApiHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Abstract base class for sql hooks.</span>
+<span class="sd"> """</span>
+ <span class="c1"># Override to provide the connection name.</span>
+ <span class="n">conn_name_attr</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="c1"># Override to have a default connection id for a particular dbHook</span>
+ <span class="n">default_conn_name</span> <span class="o">=</span> <span class="s1">'default_conn_id'</span>
+ <span class="c1"># Override if this db supports autocommit.</span>
+ <span class="n">supports_autocommit</span> <span class="o">=</span> <span class="kc">False</span>
+ <span class="c1"># Override with the object that exposes the connect method</span>
+ <span class="n">connector</span> <span class="o">=</span> <span class="kc">None</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_name_attr</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"conn_name_attr is not defined"</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="nb">len</span><span class="p">(</span><span class="n">args</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
+ <span class="nb">setattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_name_attr</span><span class="p">,</span> <span class="n">args</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
+ <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_name_attr</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">kwargs</span><span class="p">:</span>
+ <span class="nb">setattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_name_attr</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_conn_name</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="nb">setattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_name_attr</span><span class="p">,</span> <span class="n">kwargs</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_name_attr</span><span class="p">])</span>
+
+<div class="viewcode-block" id="DbApiHook.get_conn"><a class="viewcode-back" href="../../../code.html#airflow.hooks.dbapi_hook.DbApiHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""Returns a connection object</span>
+<span class="sd"> """</span>
+ <span class="n">db</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_name_attr</span><span class="p">))</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connector</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span>
+ <span class="n">host</span><span class="o">=</span><span class="n">db</span><span class="o">.</span><span class="n">host</span><span class="p">,</span>
+ <span class="n">port</span><span class="o">=</span><span class="n">db</span><span class="o">.</span><span class="n">port</span><span class="p">,</span>
+ <span class="n">username</span><span class="o">=</span><span class="n">db</span><span class="o">.</span><span class="n">login</span><span class="p">,</span>
+ <span class="n">schema</span><span class="o">=</span><span class="n">db</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span></div>
+
+ <span class="k">def</span> <span class="nf">get_uri</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_name_attr</span><span class="p">))</span>
+ <span class="n">login</span> <span class="o">=</span> <span class="s1">''</span>
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">:</span>
+ <span class="n">login</span> <span class="o">=</span> <span class="s1">'</span><span class="si">{conn.login}</span><span class="s1">:</span><span class="si">{conn.password}</span><span class="s1">@'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">conn</span><span class="o">=</span><span class="n">conn</span><span class="p">)</span>
+ <span class="n">host</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span>
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">host</span> <span class="o">+=</span> <span class="s1">':</span><span class="si">{port}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">port</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">)</span>
+ <span class="k">return</span> <span class="s1">'</span><span class="si">{conn.conn_type}</span><span class="s1">://</span><span class="si">{login}{host}</span><span class="s1">/</span><span class="si">{conn.schema}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">conn</span><span class="o">=</span><span class="n">conn</span><span class="p">,</span> <span class="n">login</span><span class="o">=</span><span class="n">login</span><span class="p">,</span> <span class="n">host</span><span class="o">=</span><span class="n">host</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">get_sqlalchemy_engine</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">engine_kwargs</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">engine_kwargs</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">engine_kwargs</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="k">return</span> <span class="n">create_engine</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_uri</span><span class="p">(),</span> <span class="o">**</span><span class="n">engine_kwargs</span><span class="p">)</span>
+
+<div class="viewcode-block" id="DbApiHook.get_pandas_df"><a class="viewcode-back" href="../../../code.html#airflow.hooks.dbapi_hook.DbApiHook.get_pandas_df">[docs]</a> <span class="k">def</span> <span class="nf">get_pandas_df</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Executes the sql and returns a pandas dataframe</span>
+
+<span class="sd"> :param sql: the sql statement to be executed (str) or a list of</span>
+<span class="sd"> sql statements to execute</span>
+<span class="sd"> :type sql: str or list</span>
+<span class="sd"> :param parameters: The parameters to render the SQL query with.</span>
+<span class="sd"> :type parameters: mapping or iterable</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o"><</span> <span class="mi">3</span><span class="p">:</span>
+ <span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span>
+ <span class="kn">import</span> <span class="nn">pandas.io.sql</span> <span class="k">as</span> <span class="nn">psql</span>
+
+ <span class="k">with</span> <span class="n">closing</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">())</span> <span class="k">as</span> <span class="n">conn</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">psql</span><span class="o">.</span><span class="n">read_sql</span><span class="p">(</span><span class="n">sql</span><span class="p">,</span> <span class="n">con</span><span class="o">=</span><span class="n">conn</span><span class="p">,</span> <span class="n">params</span><span class="o">=</span><span class="n">parameters</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="DbApiHook.get_records"><a class="viewcode-back" href="../../../code.html#airflow.hooks.dbapi_hook.DbApiHook.get_records">[docs]</a> <span class="k">def</span> <span class="nf">get_records</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Executes the sql and returns a set of records.</span>
+
+<span class="sd"> :param sql: the sql statement to be executed (str) or a list of</span>
+<span class="sd"> sql statements to execute</span>
+<span class="sd"> :type sql: str or list</span>
+<span class="sd"> :param parameters: The parameters to render the SQL query with.</span>
+<span class="sd"> :type parameters: mapping or iterable</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o"><</span> <span class="mi">3</span><span class="p">:</span>
+ <span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span>
+
+ <span class="k">with</span> <span class="n">closing</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">())</span> <span class="k">as</span> <span class="n">conn</span><span class="p">:</span>
+ <span class="k">with</span> <span class="n">closing</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">())</span> <span class="k">as</span> <span class="n">cur</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">parameters</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql</span><span class="p">,</span> <span class="n">parameters</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">cur</span><span class="o">.</span><span class="n">fetchall</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="DbApiHook.get_first"><a class="viewcode-back" href="../../../code.html#airflow.hooks.dbapi_hook.DbApiHook.get_first">[docs]</a> <span class="k">def</span> <span class="nf">get_first</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Executes the sql and returns the first resulting row.</span>
+
+<span class="sd"> :param sql: the sql statement to be executed (str) or a list of</span>
+<span class="sd"> sql statements to execute</span>
+<span class="sd"> :type sql: str or list</span>
+<span class="sd"> :param parameters: The parameters to render the SQL query with.</span>
+<span class="sd"> :type parameters: mapping or iterable</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o"><</span> <span class="mi">3</span><span class="p">:</span>
+ <span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span>
+
+ <span class="k">with</span> <span class="n">closing</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">())</span> <span class="k">as</span> <span class="n">conn</span><span class="p">:</span>
+ <span class="k">with</span> <span class="n">closing</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">())</span> <span class="k">as</span> <span class="n">cur</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">parameters</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql</span><span class="p">,</span> <span class="n">parameters</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">cur</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="DbApiHook.run"><a class="viewcode-back" href="../../../code.html#airflow.hooks.dbapi_hook.DbApiHook.run">[docs]</a> <span class="k">def</span> <span class="nf">run</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">autocommit</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Runs a command or a list of commands. Pass a list of sql</span>
+<span class="sd"> statements to the sql parameter to get them to execute</span>
+<span class="sd"> sequentially</span>
+
+<span class="sd"> :param sql: the sql statement to be executed (str) or a list of</span>
+<span class="sd"> sql statements to execute</span>
+<span class="sd"> :type sql: str or list</span>
+<span class="sd"> :param autocommit: What to set the connection's autocommit setting to</span>
+<span class="sd"> before executing the query.</span>
+<span class="sd"> :type autocommit: bool</span>
+<span class="sd"> :param parameters: The parameters to render the SQL query with.</span>
+<span class="sd"> :type parameters: mapping or iterable</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">sql</span><span class="p">,</span> <span class="n">basestring</span><span class="p">):</span>
+ <span class="n">sql</span> <span class="o">=</span> <span class="p">[</span><span class="n">sql</span><span class="p">]</span>
+
+ <span class="k">with</span> <span class="n">closing</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">())</span> <span class="k">as</span> <span class="n">conn</span><span class="p">:</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">supports_autocommit</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">set_autocommit</span><span class="p">(</span><span class="n">conn</span><span class="p">,</span> <span class="n">autocommit</span><span class="p">)</span>
+
+ <span class="k">with</span> <span class="n">closing</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">())</span> <span class="k">as</span> <span class="n">cur</span><span class="p">:</span>
+ <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">sql</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o"><</span> <span class="mi">3</span><span class="p">:</span>
+ <span class="n">s</span> <span class="o">=</span> <span class="n">s</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">s</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">parameters</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="n">parameters</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">s</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">conn</span><span class="p">,</span> <span class="s1">'autocommit'</span><span class="p">,</span> <span class="kc">False</span><span class="p">):</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="DbApiHook.set_autocommit"><a class="viewcode-back" href="../../../code.html#airflow.hooks.dbapi_hook.DbApiHook.set_autocommit">[docs]</a> <span class="k">def</span> <span class="nf">set_autocommit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn</span><span class="p">,</span> <span class="n">autocommit</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Sets the autocommit flag on the connection</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">supports_autocommit</span> <span class="ow">and</span> <span class="n">autocommit</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+ <span class="p">(</span><span class="s2">"</span><span class="si">%s</span><span class="s2"> connection doesn't support "</span>
+ <span class="s2">"autocommit but autocommit activated."</span><span class="p">),</span>
+ <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_name_attr</span><span class="p">))</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">autocommit</span> <span class="o">=</span> <span class="n">autocommit</span></div>
+
+<div class="viewcode-block" id="DbApiHook.get_cursor"><a class="viewcode-back" href="../../../code.html#airflow.hooks.dbapi_hook.DbApiHook.get_cursor">[docs]</a> <span class="k">def</span> <span class="nf">get_cursor</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns a cursor</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="DbApiHook.insert_rows"><a class="viewcode-back" href="../../../code.html#airflow.hooks.dbapi_hook.DbApiHook.insert_rows">[docs]</a> <span class="k">def</span> <span class="nf">insert_rows</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">rows</span><span class="p">,</span> <span class="n">target_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">commit_every</span><span class="o">=</span><span class="mi">1000</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> A generic way to insert a set of tuples into a table,</span>
+<span class="sd"> a new transaction is created every commit_every rows</span>
+
+<span class="sd"> :param table: Name of the target table</span>
+<span class="sd"> :type table: str</span>
+<span class="sd"> :param rows: The rows to insert into the table</span>
+<span class="sd"> :type rows: iterable of tuples</span>
+<span class="sd"> :param target_fields: The names of the columns to fill in the table</span>
+<span class="sd"> :type target_fields: iterable of strings</span>
+<span class="sd"> :param commit_every: The maximum number of rows to insert in one</span>
+<span class="sd"> transaction. Set to 0 to insert all rows in one transaction.</span>
+<span class="sd"> :type commit_every: int</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">target_fields</span><span class="p">:</span>
+ <span class="n">target_fields</span> <span class="o">=</span> <span class="s2">", "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">target_fields</span><span class="p">)</span>
+ <span class="n">target_fields</span> <span class="o">=</span> <span class="s2">"(</span><span class="si">{}</span><span class="s2">)"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">target_fields</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">target_fields</span> <span class="o">=</span> <span class="s1">''</span>
+ <span class="n">i</span> <span class="o">=</span> <span class="mi">0</span>
+ <span class="k">with</span> <span class="n">closing</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">())</span> <span class="k">as</span> <span class="n">conn</span><span class="p">:</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">supports_autocommit</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">set_autocommit</span><span class="p">(</span><span class="n">conn</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
+
+ <span class="n">conn</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
+
+ <span class="k">with</span> <span class="n">closing</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">())</span> <span class="k">as</span> <span class="n">cur</span><span class="p">:</span>
+ <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">row</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">rows</span><span class="p">,</span> <span class="mi">1</span><span class="p">):</span>
+ <span class="n">l</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="k">for</span> <span class="n">cell</span> <span class="ow">in</span> <span class="n">row</span><span class="p">:</span>
+ <span class="n">l</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_serialize_cell</span><span class="p">(</span><span class="n">cell</span><span class="p">,</span> <span class="n">conn</span><span class="p">))</span>
+ <span class="n">values</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">l</span><span class="p">)</span>
+ <span class="n">placeholders</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"</span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="p">]</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">values</span><span class="p">)</span>
+ <span class="n">sql</span> <span class="o">=</span> <span class="s2">"INSERT INTO </span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2"> VALUES (</span><span class="si">{2}</span><span class="s2">)"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">table</span><span class="p">,</span>
+ <span class="n">target_fields</span><span class="p">,</span>
+ <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">placeholders</span><span class="p">))</span>
+ <span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql</span><span class="p">,</span> <span class="n">values</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">commit_every</span> <span class="ow">and</span> <span class="n">i</span> <span class="o">%</span> <span class="n">commit_every</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s2">"Loaded </span><span class="si">{i}</span><span class="s2"> into </span><span class="si">{table}</span><span class="s2"> rows so far"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
+ <span class="p">)</span>
+
+ <span class="n">conn</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s2">"Done loading. Loaded a total of </span><span class="si">{i}</span><span class="s2"> rows"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span></div>
+
+ <span class="nd">@staticmethod</span>
+ <span class="k">def</span> <span class="nf">_serialize_cell</span><span class="p">(</span><span class="n">cell</span><span class="p">,</span> <span class="n">conn</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns the SQL literal of the cell as a string.</span>
+
+<span class="sd"> :param cell: The cell to insert into the table</span>
+<span class="sd"> :type cell: object</span>
+<span class="sd"> :param conn: The database connection</span>
+<span class="sd"> :type conn: connection object</span>
+<span class="sd"> :return: The serialized cell</span>
+<span class="sd"> :rtype: str</span>
+<span class="sd"> """</span>
+
+ <span class="k">if</span> <span class="n">cell</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">None</span>
+ <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">cell</span><span class="p">,</span> <span class="n">datetime</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">cell</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
+ <span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">cell</span><span class="p">)</span>
+
+<div class="viewcode-block" id="DbApiHook.bulk_dump"><a class="viewcode-back" href="../../../code.html#airflow.hooks.dbapi_hook.DbApiHook.bulk_dump">[docs]</a> <span class="k">def</span> <span class="nf">bulk_dump</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">tmp_file</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Dumps a database table into a tab-delimited file</span>
+
+<span class="sd"> :param table: The name of the source table</span>
+<span class="sd"> :type table: str</span>
+<span class="sd"> :param tmp_file: The path of the target file</span>
+<span class="sd"> :type tmp_file: str</span>
+<span class="sd"> """</span>
+ <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="DbApiHook.bulk_load"><a class="viewcode-back" href="../../../code.html#airflow.hooks.dbapi_hook.DbApiHook.bulk_load">[docs]</a> <span class="k">def</span> <span class="nf">bulk_load</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">tmp_file</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Loads a tab-delimited file into a database table</span>
+
+<span class="sd"> :param table: The name of the target table</span>
+<span class="sd"> :type table: str</span>
+<span class="sd"> :param tmp_file: The path of the file to load into the table</span>
+<span class="sd"> :type tmp_file: str</span>
+<span class="sd"> """</span>
+ <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">()</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[40/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/spark_jdbc_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/spark_jdbc_hook.html b/_modules/airflow/contrib/hooks/spark_jdbc_hook.html
new file mode 100644
index 0000000..c22b1e5
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/spark_jdbc_hook.html
@@ -0,0 +1,481 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.spark_jdbc_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.spark_jdbc_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.spark_jdbc_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+<span class="kn">import</span> <span class="nn">os</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.spark_submit_hook</span> <span class="k">import</span> <span class="n">SparkSubmitHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+
+
+<div class="viewcode-block" id="SparkJDBCHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.spark_jdbc_hook.SparkJDBCHook">[docs]</a><span class="k">class</span> <span class="nc">SparkJDBCHook</span><span class="p">(</span><span class="n">SparkSubmitHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> This hook extends the SparkSubmitHook specifically for performing data</span>
+<span class="sd"> transfers to/from JDBC-based databases with Apache Spark.</span>
+
+<span class="sd"> :param spark_app_name: Name of the job (default airflow-spark-jdbc)</span>
+<span class="sd"> :type spark_app_name: str</span>
+<span class="sd"> :param spark_conn_id: Connection id as configured in Airflow administration</span>
+<span class="sd"> :type spark_conn_id: str</span>
+<span class="sd"> :param spark_conf: Any additional Spark configuration properties</span>
+<span class="sd"> :type spark_conf: dict</span>
+<span class="sd"> :param spark_py_files: Additional python files used (.zip, .egg, or .py)</span>
+<span class="sd"> :type spark_py_files: str</span>
+<span class="sd"> :param spark_files: Additional files to upload to the container running the job</span>
+<span class="sd"> :type spark_files: str</span>
+<span class="sd"> :param spark_jars: Additional jars to upload and add to the driver and</span>
+<span class="sd"> executor classpath</span>
+<span class="sd"> :type spark_jars: str</span>
+<span class="sd"> :param num_executors: number of executor to run. This should be set so as to manage</span>
+<span class="sd"> the number of connections made with the JDBC database</span>
+<span class="sd"> :type num_executors: int</span>
+<span class="sd"> :param executor_cores: Number of cores per executor</span>
+<span class="sd"> :type executor_cores: int</span>
+<span class="sd"> :param executor_memory: Memory per executor (e.g. 1000M, 2G)</span>
+<span class="sd"> :type executor_memory: str</span>
+<span class="sd"> :param driver_memory: Memory allocated to the driver (e.g. 1000M, 2G)</span>
+<span class="sd"> :type driver_memory: str</span>
+<span class="sd"> :param verbose: Whether to pass the verbose flag to spark-submit for debugging</span>
+<span class="sd"> :type verbose: bool</span>
+<span class="sd"> :param keytab: Full path to the file that contains the keytab</span>
+<span class="sd"> :type keytab: str</span>
+<span class="sd"> :param principal: The name of the kerberos principal used for keytab</span>
+<span class="sd"> :type principal: str</span>
+<span class="sd"> :param cmd_type: Which way the data should flow. 2 possible values:</span>
+<span class="sd"> spark_to_jdbc: data written by spark from metastore to jdbc</span>
+<span class="sd"> jdbc_to_spark: data written by spark from jdbc to metastore</span>
+<span class="sd"> :type cmd_type: str</span>
+<span class="sd"> :param jdbc_table: The name of the JDBC table</span>
+<span class="sd"> :type jdbc_table: str</span>
+<span class="sd"> :param jdbc_conn_id: Connection id used for connection to JDBC database</span>
+<span class="sd"> :type: jdbc_conn_id: str</span>
+<span class="sd"> :param jdbc_driver: Name of the JDBC driver to use for the JDBC connection. This</span>
+<span class="sd"> driver (usually a jar) should be passed in the 'jars' parameter</span>
+<span class="sd"> :type jdbc_driver: str</span>
+<span class="sd"> :param metastore_table: The name of the metastore table,</span>
+<span class="sd"> :type metastore_table: str</span>
+<span class="sd"> :param jdbc_truncate: (spark_to_jdbc only) Whether or not Spark should truncate or</span>
+<span class="sd"> drop and recreate the JDBC table. This only takes effect if</span>
+<span class="sd"> 'save_mode' is set to Overwrite. Also, if the schema is</span>
+<span class="sd"> different, Spark cannot truncate, and will drop and recreate</span>
+<span class="sd"> :type jdbc_truncate: bool</span>
+<span class="sd"> :param save_mode: The Spark save-mode to use (e.g. overwrite, append, etc.)</span>
+<span class="sd"> :type save_mode: str</span>
+<span class="sd"> :param save_format: (jdbc_to_spark-only) The Spark save-format to use (e.g. parquet)</span>
+<span class="sd"> :type save_format: str</span>
+<span class="sd"> :param batch_size: (spark_to_jdbc only) The size of the batch to insert per round</span>
+<span class="sd"> trip to the JDBC database. Defaults to 1000</span>
+<span class="sd"> :type batch_size: int</span>
+<span class="sd"> :param fetch_size: (jdbc_to_spark only) The size of the batch to fetch per round trip</span>
+<span class="sd"> from the JDBC database. Default depends on the JDBC driver</span>
+<span class="sd"> :type fetch_size: int</span>
+<span class="sd"> :param num_partitions: The maximum number of partitions that can be used by Spark</span>
+<span class="sd"> simultaneously, both for spark_to_jdbc and jdbc_to_spark</span>
+<span class="sd"> operations. This will also cap the number of JDBC connections</span>
+<span class="sd"> that can be opened</span>
+<span class="sd"> :type num_partitions: int</span>
+<span class="sd"> :param partition_column: (jdbc_to_spark-only) A numeric column to be used to</span>
+<span class="sd"> partition the metastore table by. If specified, you must</span>
+<span class="sd"> also specify:</span>
+<span class="sd"> num_partitions, lower_bound, upper_bound</span>
+<span class="sd"> :type partition_column: str</span>
+<span class="sd"> :param lower_bound: (jdbc_to_spark-only) Lower bound of the range of the numeric</span>
+<span class="sd"> partition column to fetch. If specified, you must also specify:</span>
+<span class="sd"> num_partitions, partition_column, upper_bound</span>
+<span class="sd"> :type lower_bound: int</span>
+<span class="sd"> :param upper_bound: (jdbc_to_spark-only) Upper bound of the range of the numeric</span>
+<span class="sd"> partition column to fetch. If specified, you must also specify:</span>
+<span class="sd"> num_partitions, partition_column, lower_bound</span>
+<span class="sd"> :type upper_bound: int</span>
+<span class="sd"> :param create_table_column_types: (spark_to_jdbc-only) The database column data types</span>
+<span class="sd"> to use instead of the defaults, when creating the</span>
+<span class="sd"> table. Data type information should be specified in</span>
+<span class="sd"> the same format as CREATE TABLE columns syntax</span>
+<span class="sd"> (e.g: "name CHAR(64), comments VARCHAR(1024)").</span>
+<span class="sd"> The specified types should be valid spark sql data</span>
+<span class="sd"> types.</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">spark_app_name</span><span class="o">=</span><span class="s1">'airflow-spark-jdbc'</span><span class="p">,</span>
+ <span class="n">spark_conn_id</span><span class="o">=</span><span class="s1">'spark-default'</span><span class="p">,</span>
+ <span class="n">spark_conf</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">spark_py_files</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">spark_files</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">spark_jars</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">num_executors</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">executor_cores</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">executor_memory</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">driver_memory</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">principal</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">keytab</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">cmd_type</span><span class="o">=</span><span class="s1">'spark_to_jdbc'</span><span class="p">,</span>
+ <span class="n">jdbc_table</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">jdbc_conn_id</span><span class="o">=</span><span class="s1">'jdbc-default'</span><span class="p">,</span>
+ <span class="n">jdbc_driver</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">metastore_table</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">jdbc_truncate</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">save_mode</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">save_format</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">batch_size</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">fetch_size</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">num_partitions</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">partition_column</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">lower_bound</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">upper_bound</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">create_table_column_types</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span>
+ <span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">SparkJDBCHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="n">spark_app_name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_conn_id</span> <span class="o">=</span> <span class="n">spark_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_conf</span> <span class="o">=</span> <span class="n">spark_conf</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_py_files</span> <span class="o">=</span> <span class="n">spark_py_files</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_files</span> <span class="o">=</span> <span class="n">spark_files</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_jars</span> <span class="o">=</span> <span class="n">spark_jars</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_num_executors</span> <span class="o">=</span> <span class="n">num_executors</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_executor_cores</span> <span class="o">=</span> <span class="n">executor_cores</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_executor_memory</span> <span class="o">=</span> <span class="n">executor_memory</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_driver_memory</span> <span class="o">=</span> <span class="n">driver_memory</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_verbose</span> <span class="o">=</span> <span class="n">verbose</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_keytab</span> <span class="o">=</span> <span class="n">keytab</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_principal</span> <span class="o">=</span> <span class="n">principal</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_cmd_type</span> <span class="o">=</span> <span class="n">cmd_type</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_table</span> <span class="o">=</span> <span class="n">jdbc_table</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_conn_id</span> <span class="o">=</span> <span class="n">jdbc_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_driver</span> <span class="o">=</span> <span class="n">jdbc_driver</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_metastore_table</span> <span class="o">=</span> <span class="n">metastore_table</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_truncate</span> <span class="o">=</span> <span class="n">jdbc_truncate</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_save_mode</span> <span class="o">=</span> <span class="n">save_mode</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_save_format</span> <span class="o">=</span> <span class="n">save_format</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_batch_size</span> <span class="o">=</span> <span class="n">batch_size</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_fetch_size</span> <span class="o">=</span> <span class="n">fetch_size</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_num_partitions</span> <span class="o">=</span> <span class="n">num_partitions</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_partition_column</span> <span class="o">=</span> <span class="n">partition_column</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_lower_bound</span> <span class="o">=</span> <span class="n">lower_bound</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_upper_bound</span> <span class="o">=</span> <span class="n">upper_bound</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_create_table_column_types</span> <span class="o">=</span> <span class="n">create_table_column_types</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_connection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_resolve_jdbc_connection</span><span class="p">()</span>
+
+ <span class="k">def</span> <span class="nf">_resolve_jdbc_connection</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="n">conn_data</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'url'</span><span class="p">:</span> <span class="s1">''</span><span class="p">,</span>
+ <span class="s1">'schema'</span><span class="p">:</span> <span class="s1">''</span><span class="p">,</span>
+ <span class="s1">'conn_prefix'</span><span class="p">:</span> <span class="s1">''</span><span class="p">,</span>
+ <span class="s1">'user'</span><span class="p">:</span> <span class="s1">''</span><span class="p">,</span>
+ <span class="s1">'password'</span><span class="p">:</span> <span class="s1">''</span>
+ <span class="p">}</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_conn_id</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">:</span>
+ <span class="n">conn_data</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{}</span><span class="s2">:</span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">conn_data</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span>
+ <span class="n">conn_data</span><span class="p">[</span><span class="s1">'schema'</span><span class="p">]</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">schema</span>
+ <span class="n">conn_data</span><span class="p">[</span><span class="s1">'user'</span><span class="p">]</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span>
+ <span class="n">conn_data</span><span class="p">[</span><span class="s1">'password'</span><span class="p">]</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span>
+ <span class="n">extra</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span>
+ <span class="n">conn_data</span><span class="p">[</span><span class="s1">'conn_prefix'</span><span class="p">]</span> <span class="o">=</span> <span class="n">extra</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'conn_prefix'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
+ <span class="k">except</span> <span class="n">AirflowException</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
+ <span class="s2">"Could not load jdbc connection string </span><span class="si">%s</span><span class="s2">, defaulting to </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_conn_id</span><span class="p">,</span> <span class="s2">""</span>
+ <span class="p">)</span>
+ <span class="k">return</span> <span class="n">conn_data</span>
+
+ <span class="k">def</span> <span class="nf">_build_jdbc_application_arguments</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">jdbc_conn</span><span class="p">):</span>
+ <span class="n">arguments</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="n">arguments</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"-cmdType"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_cmd_type</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_connection</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]:</span>
+ <span class="n">arguments</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'-url'</span><span class="p">,</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">/</span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">jdbc_conn</span><span class="p">[</span><span class="s1">'conn_prefix'</span><span class="p">],</span> <span class="n">jdbc_conn</span><span class="p">[</span><span class="s1">'url'</span><span class="p">],</span> <span class="n">jdbc_conn</span><span class="p">[</span><span class="s1">'schema'</span><span class="p">]</span>
+ <span class="p">)]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_connection</span><span class="p">[</span><span class="s1">'user'</span><span class="p">]:</span>
+ <span class="n">arguments</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'-user'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_connection</span><span class="p">[</span><span class="s1">'user'</span><span class="p">]]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_connection</span><span class="p">[</span><span class="s1">'password'</span><span class="p">]:</span>
+ <span class="n">arguments</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'-password'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_connection</span><span class="p">[</span><span class="s1">'password'</span><span class="p">]]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_metastore_table</span><span class="p">:</span>
+ <span class="n">arguments</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'-metastoreTable'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_metastore_table</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_table</span><span class="p">:</span>
+ <span class="n">arguments</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'-jdbcTable'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_table</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_truncate</span><span class="p">:</span>
+ <span class="n">arguments</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'-jdbcTruncate'</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_truncate</span><span class="p">)]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_driver</span><span class="p">:</span>
+ <span class="n">arguments</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'-jdbcDriver'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_driver</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_batch_size</span><span class="p">:</span>
+ <span class="n">arguments</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'-batchsize'</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_batch_size</span><span class="p">)]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_fetch_size</span><span class="p">:</span>
+ <span class="n">arguments</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'-fetchsize'</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_fetch_size</span><span class="p">)]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_num_partitions</span><span class="p">:</span>
+ <span class="n">arguments</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'-numPartitions'</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_num_partitions</span><span class="p">)]</span>
+ <span class="k">if</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_partition_column</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_lower_bound</span> <span class="ow">and</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_upper_bound</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_num_partitions</span><span class="p">):</span>
+ <span class="c1"># these 3 parameters need to be used all together to take effect.</span>
+ <span class="n">arguments</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'-partitionColumn'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_partition_column</span><span class="p">,</span>
+ <span class="s1">'-lowerBound'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_lower_bound</span><span class="p">,</span>
+ <span class="s1">'-upperBound'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_upper_bound</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_save_mode</span><span class="p">:</span>
+ <span class="n">arguments</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'-saveMode'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_save_mode</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_save_format</span><span class="p">:</span>
+ <span class="n">arguments</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'-saveFormat'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_save_format</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_create_table_column_types</span><span class="p">:</span>
+ <span class="n">arguments</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'-createTableColumnTypes'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_create_table_column_types</span><span class="p">]</span>
+ <span class="k">return</span> <span class="n">arguments</span>
+
+ <span class="k">def</span> <span class="nf">submit_jdbc_job</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_application_args</span> <span class="o">=</span> \
+ <span class="bp">self</span><span class="o">.</span><span class="n">_build_jdbc_application_arguments</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdbc_connection</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">application</span><span class="o">=</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="vm">__file__</span><span class="p">))</span> <span class="o">+</span>
+ <span class="s2">"/spark_jdbc_script.py"</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">pass</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/spark_sql_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/spark_sql_hook.html b/_modules/airflow/contrib/hooks/spark_sql_hook.html
new file mode 100644
index 0000000..08d3147
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/spark_sql_hook.html
@@ -0,0 +1,396 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.spark_sql_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.spark_sql_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.spark_sql_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+<span class="kn">import</span> <span class="nn">subprocess</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+
+<div class="viewcode-block" id="SparkSqlHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.spark_sql_hook.SparkSqlHook">[docs]</a><span class="k">class</span> <span class="nc">SparkSqlHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> This hook is a wrapper around the spark-sql binary. It requires that the</span>
+<span class="sd"> "spark-sql" binary is in the PATH.</span>
+<span class="sd"> :param sql: The SQL query to execute</span>
+<span class="sd"> :type sql: str</span>
+<span class="sd"> :param conf: arbitrary Spark configuration property</span>
+<span class="sd"> :type conf: str (format: PROP=VALUE)</span>
+<span class="sd"> :param conn_id: connection_id string</span>
+<span class="sd"> :type conn_id: str</span>
+<span class="sd"> :param total_executor_cores: (Standalone & Mesos only) Total cores for all executors (Default: all the available cores on the worker)</span>
+<span class="sd"> :type total_executor_cores: int</span>
+<span class="sd"> :param executor_cores: (Standalone & YARN only) Number of cores per executor (Default: 2)</span>
+<span class="sd"> :type executor_cores: int</span>
+<span class="sd"> :param executor_memory: Memory per executor (e.g. 1000M, 2G) (Default: 1G)</span>
+<span class="sd"> :type executor_memory: str</span>
+<span class="sd"> :param keytab: Full path to the file that contains the keytab</span>
+<span class="sd"> :type keytab: str</span>
+<span class="sd"> :param master: spark://host:port, mesos://host:port, yarn, or local</span>
+<span class="sd"> :type master: str</span>
+<span class="sd"> :param name: Name of the job.</span>
+<span class="sd"> :type name: str</span>
+<span class="sd"> :param num_executors: Number of executors to launch</span>
+<span class="sd"> :type num_executors: int</span>
+<span class="sd"> :param verbose: Whether to pass the verbose flag to spark-sql</span>
+<span class="sd"> :type verbose: bool</span>
+<span class="sd"> :param yarn_queue: The YARN queue to submit to (Default: "default")</span>
+<span class="sd"> :type yarn_queue: str</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">sql</span><span class="p">,</span>
+ <span class="n">conf</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">conn_id</span><span class="o">=</span><span class="s1">'spark_sql_default'</span><span class="p">,</span>
+ <span class="n">total_executor_cores</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">executor_cores</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">executor_memory</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">keytab</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">principal</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">master</span><span class="o">=</span><span class="s1">'yarn'</span><span class="p">,</span>
+ <span class="n">name</span><span class="o">=</span><span class="s1">'default-name'</span><span class="p">,</span>
+ <span class="n">num_executors</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">verbose</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+ <span class="n">yarn_queue</span><span class="o">=</span><span class="s1">'default'</span>
+ <span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_sql</span> <span class="o">=</span> <span class="n">sql</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_conf</span> <span class="o">=</span> <span class="n">conf</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">conn_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_total_executor_cores</span> <span class="o">=</span> <span class="n">total_executor_cores</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_executor_cores</span> <span class="o">=</span> <span class="n">executor_cores</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_executor_memory</span> <span class="o">=</span> <span class="n">executor_memory</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_keytab</span> <span class="o">=</span> <span class="n">keytab</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_principal</span> <span class="o">=</span> <span class="n">principal</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_master</span> <span class="o">=</span> <span class="n">master</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="n">name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_num_executors</span> <span class="o">=</span> <span class="n">num_executors</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_verbose</span> <span class="o">=</span> <span class="n">verbose</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_yarn_queue</span> <span class="o">=</span> <span class="n">yarn_queue</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_sp</span> <span class="o">=</span> <span class="kc">None</span>
+
+ <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">pass</span>
+
+ <span class="k">def</span> <span class="nf">_prepare_command</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cmd</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Construct the spark-sql command to execute. Verbose output is enabled</span>
+<span class="sd"> as default.</span>
+<span class="sd"> :param cmd: command to append to the spark-sql command</span>
+<span class="sd"> :type cmd: str</span>
+<span class="sd"> :return: full command to be executed</span>
+<span class="sd"> """</span>
+ <span class="n">connection_cmd</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"spark-sql"</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_conf</span><span class="p">:</span>
+ <span class="k">for</span> <span class="n">conf_el</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_conf</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">):</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--conf"</span><span class="p">,</span> <span class="n">conf_el</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_total_executor_cores</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--total-executor-cores"</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_total_executor_cores</span><span class="p">)]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_executor_cores</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--executor-cores"</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_executor_cores</span><span class="p">)]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_executor_memory</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--executor-memory"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_executor_memory</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_keytab</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--keytab"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_keytab</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_principal</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--principal"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_principal</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_num_executors</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--num-executors"</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_num_executors</span><span class="p">)]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sql</span><span class="p">:</span>
+ <span class="n">sql</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sql</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
+ <span class="k">if</span> <span class="n">sql</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">".sql"</span><span class="p">)</span> <span class="ow">or</span> <span class="n">sql</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">".hql"</span><span class="p">):</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"-f"</span><span class="p">,</span> <span class="n">sql</span><span class="p">]</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"-e"</span><span class="p">,</span> <span class="n">sql</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_master</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--master"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_master</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_name</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--name"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_name</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_verbose</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--verbose"</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_yarn_queue</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--queue"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_yarn_queue</span><span class="p">]</span>
+
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="n">cmd</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Spark-Sql cmd: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">connection_cmd</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">connection_cmd</span>
+
+<div class="viewcode-block" id="SparkSqlHook.run_query"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.spark_sql_hook.SparkSqlHook.run_query">[docs]</a> <span class="k">def</span> <span class="nf">run_query</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cmd</span><span class="o">=</span><span class="s2">""</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Remote Popen (actually execute the Spark-sql query)</span>
+
+<span class="sd"> :param cmd: command to remotely execute</span>
+<span class="sd"> :param kwargs: extra arguments to Popen (see subprocess.Popen)</span>
+<span class="sd"> """</span>
+ <span class="n">spark_sql_cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_command</span><span class="p">(</span><span class="n">cmd</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_sp</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">Popen</span><span class="p">(</span><span class="n">spark_sql_cmd</span><span class="p">,</span>
+ <span class="n">stdout</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span>
+ <span class="n">stderr</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">STDOUT</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="nb">iter</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_sp</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">readline</span><span class="p">,</span> <span class="s1">''</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">line</span><span class="p">)</span>
+
+ <span class="n">returncode</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sp</span><span class="o">.</span><span class="n">wait</span><span class="p">()</span>
+
+ <span class="k">if</span> <span class="n">returncode</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s2">"Cannot execute </span><span class="si">{}</span><span class="s2"> on </span><span class="si">{}</span><span class="s2">. Process exit code: </span><span class="si">{}</span><span class="s2">."</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">cmd</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_conn</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> <span class="n">returncode</span>
+ <span class="p">)</span>
+ <span class="p">)</span></div>
+
+ <span class="k">def</span> <span class="nf">kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sp</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sp</span><span class="o">.</span><span class="n">poll</span><span class="p">()</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Killing the Spark-Sql job"</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_sp</span><span class="o">.</span><span class="n">kill</span><span class="p">()</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[44/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/gcp_pubsub_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/gcp_pubsub_hook.html b/_modules/airflow/contrib/hooks/gcp_pubsub_hook.html
new file mode 100644
index 0000000..19713d1
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/gcp_pubsub_hook.html
@@ -0,0 +1,519 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.gcp_pubsub_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.gcp_pubsub_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.gcp_pubsub_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">uuid</span> <span class="k">import</span> <span class="n">uuid4</span>
+
+<span class="kn">from</span> <span class="nn">apiclient.discovery</span> <span class="k">import</span> <span class="n">build</span>
+<span class="kn">from</span> <span class="nn">apiclient</span> <span class="k">import</span> <span class="n">errors</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="k">import</span> <span class="n">GoogleCloudBaseHook</span>
+
+
+<span class="k">def</span> <span class="nf">_format_subscription</span><span class="p">(</span><span class="n">project</span><span class="p">,</span> <span class="n">subscription</span><span class="p">):</span>
+ <span class="k">return</span> <span class="s1">'projects/</span><span class="si">{}</span><span class="s1">/subscriptions/</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">project</span><span class="p">,</span> <span class="n">subscription</span><span class="p">)</span>
+
+
+<span class="k">def</span> <span class="nf">_format_topic</span><span class="p">(</span><span class="n">project</span><span class="p">,</span> <span class="n">topic</span><span class="p">):</span>
+ <span class="k">return</span> <span class="s1">'projects/</span><span class="si">{}</span><span class="s1">/topics/</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">project</span><span class="p">,</span> <span class="n">topic</span><span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">PubSubException</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
+ <span class="k">pass</span>
+
+
+<div class="viewcode-block" id="PubSubHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.gcp_pubsub_hook.PubSubHook">[docs]</a><span class="k">class</span> <span class="nc">PubSubHook</span><span class="p">(</span><span class="n">GoogleCloudBaseHook</span><span class="p">):</span>
+ <span class="sd">"""Hook for accessing Google Pub/Sub.</span>
+
+<span class="sd"> The GCP project against which actions are applied is determined by</span>
+<span class="sd"> the project embedded in the Connection referenced by gcp_conn_id.</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span> <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">PubSubHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="o">=</span><span class="n">delegate_to</span><span class="p">)</span>
+
+<div class="viewcode-block" id="PubSubHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.gcp_pubsub_hook.PubSubHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""Returns a Pub/Sub service object.</span>
+
+<span class="sd"> :rtype: apiclient.discovery.Resource</span>
+<span class="sd"> """</span>
+ <span class="n">http_authorized</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">build</span><span class="p">(</span><span class="s1">'pubsub'</span><span class="p">,</span> <span class="s1">'v1'</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="PubSubHook.publish"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.gcp_pubsub_hook.PubSubHook.publish">[docs]</a> <span class="k">def</span> <span class="nf">publish</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project</span><span class="p">,</span> <span class="n">topic</span><span class="p">,</span> <span class="n">messages</span><span class="p">):</span>
+ <span class="sd">"""Publishes messages to a Pub/Sub topic.</span>
+
+<span class="sd"> :param project: the GCP project ID in which to publish</span>
+<span class="sd"> :type project: string</span>
+<span class="sd"> :param topic: the Pub/Sub topic to which to publish; do not</span>
+<span class="sd"> include the ``projects/{project}/topics/`` prefix.</span>
+<span class="sd"> :type topic: string</span>
+<span class="sd"> :param messages: messages to publish; if the data field in a</span>
+<span class="sd"> message is set, it should already be base64 encoded.</span>
+<span class="sd"> :type messages: list of PubSub messages; see</span>
+<span class="sd"> http://cloud.google.com/pubsub/docs/reference/rest/v1/PubsubMessage</span>
+<span class="sd"> """</span>
+ <span class="n">body</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'messages'</span><span class="p">:</span> <span class="n">messages</span><span class="p">}</span>
+ <span class="n">full_topic</span> <span class="o">=</span> <span class="n">_format_topic</span><span class="p">(</span><span class="n">project</span><span class="p">,</span> <span class="n">topic</span><span class="p">)</span>
+ <span class="n">request</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">topics</span><span class="p">()</span><span class="o">.</span><span class="n">publish</span><span class="p">(</span>
+ <span class="n">topic</span><span class="o">=</span><span class="n">full_topic</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">request</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">PubSubException</span><span class="p">(</span>
+ <span class="s1">'Error publishing to topic </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">full_topic</span><span class="p">),</span> <span class="n">e</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="PubSubHook.create_topic"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.gcp_pubsub_hook.PubSubHook.create_topic">[docs]</a> <span class="k">def</span> <span class="nf">create_topic</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project</span><span class="p">,</span> <span class="n">topic</span><span class="p">,</span> <span class="n">fail_if_exists</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+ <span class="sd">"""Creates a Pub/Sub topic, if it does not already exist.</span>
+
+<span class="sd"> :param project: the GCP project ID in which to create</span>
+<span class="sd"> the topic</span>
+<span class="sd"> :type project: string</span>
+<span class="sd"> :param topic: the Pub/Sub topic name to create; do not</span>
+<span class="sd"> include the ``projects/{project}/topics/`` prefix.</span>
+<span class="sd"> :type topic: string</span>
+<span class="sd"> :param fail_if_exists: if set, raise an exception if the topic</span>
+<span class="sd"> already exists</span>
+<span class="sd"> :type fail_if_exists: bool</span>
+<span class="sd"> """</span>
+ <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">full_topic</span> <span class="o">=</span> <span class="n">_format_topic</span><span class="p">(</span><span class="n">project</span><span class="p">,</span> <span class="n">topic</span><span class="p">)</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">service</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">topics</span><span class="p">()</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
+ <span class="n">name</span><span class="o">=</span><span class="n">full_topic</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="p">{})</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="c1"># Status code 409 indicates that the topic already exists.</span>
+ <span class="k">if</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">resp</span><span class="p">[</span><span class="s1">'status'</span><span class="p">])</span> <span class="o">==</span> <span class="s1">'409'</span><span class="p">:</span>
+ <span class="n">message</span> <span class="o">=</span> <span class="s1">'Topic already exists: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">full_topic</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">message</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">fail_if_exists</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">PubSubException</span><span class="p">(</span><span class="n">message</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">PubSubException</span><span class="p">(</span>
+ <span class="s1">'Error creating topic </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">full_topic</span><span class="p">),</span> <span class="n">e</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="PubSubHook.delete_topic"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.gcp_pubsub_hook.PubSubHook.delete_topic">[docs]</a> <span class="k">def</span> <span class="nf">delete_topic</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project</span><span class="p">,</span> <span class="n">topic</span><span class="p">,</span> <span class="n">fail_if_not_exists</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+ <span class="sd">"""Deletes a Pub/Sub topic if it exists.</span>
+
+<span class="sd"> :param project: the GCP project ID in which to delete the topic</span>
+<span class="sd"> :type project: string</span>
+<span class="sd"> :param topic: the Pub/Sub topic name to delete; do not</span>
+<span class="sd"> include the ``projects/{project}/topics/`` prefix.</span>
+<span class="sd"> :type topic: string</span>
+<span class="sd"> :param fail_if_not_exists: if set, raise an exception if the topic</span>
+<span class="sd"> does not exist</span>
+<span class="sd"> :type fail_if_not_exists: bool</span>
+<span class="sd"> """</span>
+ <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">full_topic</span> <span class="o">=</span> <span class="n">_format_topic</span><span class="p">(</span><span class="n">project</span><span class="p">,</span> <span class="n">topic</span><span class="p">)</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">service</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">topics</span><span class="p">()</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">topic</span><span class="o">=</span><span class="n">full_topic</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="c1"># Status code 409 indicates that the topic was not found</span>
+ <span class="k">if</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">resp</span><span class="p">[</span><span class="s1">'status'</span><span class="p">])</span> <span class="o">==</span> <span class="s1">'404'</span><span class="p">:</span>
+ <span class="n">message</span> <span class="o">=</span> <span class="s1">'Topic does not exist: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">full_topic</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">message</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">fail_if_not_exists</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">PubSubException</span><span class="p">(</span><span class="n">message</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">PubSubException</span><span class="p">(</span>
+ <span class="s1">'Error deleting topic </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">full_topic</span><span class="p">),</span> <span class="n">e</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="PubSubHook.create_subscription"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.gcp_pubsub_hook.PubSubHook.create_subscription">[docs]</a> <span class="k">def</span> <span class="nf">create_subscription</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">topic_project</span><span class="p">,</span> <span class="n">topic</span><span class="p">,</span> <span class="n">subscription</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">subscription_project</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">ack_deadline_secs</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span>
+ <span class="n">fail_if_exists</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+ <span class="sd">"""Creates a Pub/Sub subscription, if it does not already exist.</span>
+
+<span class="sd"> :param topic_project: the GCP project ID of the topic that the</span>
+<span class="sd"> subscription will be bound to.</span>
+<span class="sd"> :type topic_project: string</span>
+<span class="sd"> :param topic: the Pub/Sub topic name that the subscription will be bound</span>
+<span class="sd"> to create; do not include the ``projects/{project}/subscriptions/``</span>
+<span class="sd"> prefix.</span>
+<span class="sd"> :type topic: string</span>
+<span class="sd"> :param subscription: the Pub/Sub subscription name. If empty, a random</span>
+<span class="sd"> name will be generated using the uuid module</span>
+<span class="sd"> :type subscription: string</span>
+<span class="sd"> :param subscription_project: the GCP project ID where the subscription</span>
+<span class="sd"> will be created. If unspecified, ``topic_project`` will be used.</span>
+<span class="sd"> :type subscription_project: string</span>
+<span class="sd"> :param ack_deadline_secs: Number of seconds that a subscriber has to</span>
+<span class="sd"> acknowledge each message pulled from the subscription</span>
+<span class="sd"> :type ack_deadline_secs: int</span>
+<span class="sd"> :param fail_if_exists: if set, raise an exception if the topic</span>
+<span class="sd"> already exists</span>
+<span class="sd"> :type fail_if_exists: bool</span>
+<span class="sd"> :return: subscription name which will be the system-generated value if</span>
+<span class="sd"> the ``subscription`` parameter is not supplied</span>
+<span class="sd"> :rtype: string</span>
+<span class="sd"> """</span>
+ <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">full_topic</span> <span class="o">=</span> <span class="n">_format_topic</span><span class="p">(</span><span class="n">topic_project</span><span class="p">,</span> <span class="n">topic</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">subscription</span><span class="p">:</span>
+ <span class="n">subscription</span> <span class="o">=</span> <span class="s1">'sub-</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">uuid4</span><span class="p">())</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">subscription_project</span><span class="p">:</span>
+ <span class="n">subscription_project</span> <span class="o">=</span> <span class="n">topic_project</span>
+ <span class="n">full_subscription</span> <span class="o">=</span> <span class="n">_format_subscription</span><span class="p">(</span><span class="n">subscription_project</span><span class="p">,</span>
+ <span class="n">subscription</span><span class="p">)</span>
+ <span class="n">body</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'topic'</span><span class="p">:</span> <span class="n">full_topic</span><span class="p">,</span>
+ <span class="s1">'ackDeadlineSeconds'</span><span class="p">:</span> <span class="n">ack_deadline_secs</span>
+ <span class="p">}</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">service</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">subscriptions</span><span class="p">()</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
+ <span class="n">name</span><span class="o">=</span><span class="n">full_subscription</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="c1"># Status code 409 indicates that the subscription already exists.</span>
+ <span class="k">if</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">resp</span><span class="p">[</span><span class="s1">'status'</span><span class="p">])</span> <span class="o">==</span> <span class="s1">'409'</span><span class="p">:</span>
+ <span class="n">message</span> <span class="o">=</span> <span class="s1">'Subscription already exists: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">full_subscription</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">message</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">fail_if_exists</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">PubSubException</span><span class="p">(</span><span class="n">message</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">PubSubException</span><span class="p">(</span>
+ <span class="s1">'Error creating subscription </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">full_subscription</span><span class="p">),</span>
+ <span class="n">e</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">subscription</span></div>
+
+<div class="viewcode-block" id="PubSubHook.delete_subscription"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.gcp_pubsub_hook.PubSubHook.delete_subscription">[docs]</a> <span class="k">def</span> <span class="nf">delete_subscription</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project</span><span class="p">,</span> <span class="n">subscription</span><span class="p">,</span>
+ <span class="n">fail_if_not_exists</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+ <span class="sd">"""Deletes a Pub/Sub subscription, if it exists.</span>
+
+<span class="sd"> :param project: the GCP project ID where the subscription exists</span>
+<span class="sd"> :type project: string</span>
+<span class="sd"> :param subscription: the Pub/Sub subscription name to delete; do not</span>
+<span class="sd"> include the ``projects/{project}/subscriptions/`` prefix.</span>
+<span class="sd"> :type subscription: string</span>
+<span class="sd"> :param fail_if_not_exists: if set, raise an exception if the topic</span>
+<span class="sd"> does not exist</span>
+<span class="sd"> :type fail_if_not_exists: bool</span>
+<span class="sd"> """</span>
+ <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">full_subscription</span> <span class="o">=</span> <span class="n">_format_subscription</span><span class="p">(</span><span class="n">project</span><span class="p">,</span> <span class="n">subscription</span><span class="p">)</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">service</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">subscriptions</span><span class="p">()</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span>
+ <span class="n">subscription</span><span class="o">=</span><span class="n">full_subscription</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="c1"># Status code 404 indicates that the subscription was not found</span>
+ <span class="k">if</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">resp</span><span class="p">[</span><span class="s1">'status'</span><span class="p">])</span> <span class="o">==</span> <span class="s1">'404'</span><span class="p">:</span>
+ <span class="n">message</span> <span class="o">=</span> <span class="s1">'Subscription does not exist: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">full_subscription</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">message</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">fail_if_not_exists</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">PubSubException</span><span class="p">(</span><span class="n">message</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">PubSubException</span><span class="p">(</span>
+ <span class="s1">'Error deleting subscription </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">full_subscription</span><span class="p">),</span>
+ <span class="n">e</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="PubSubHook.pull"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.gcp_pubsub_hook.PubSubHook.pull">[docs]</a> <span class="k">def</span> <span class="nf">pull</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project</span><span class="p">,</span> <span class="n">subscription</span><span class="p">,</span> <span class="n">max_messages</span><span class="p">,</span>
+ <span class="n">return_immediately</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+ <span class="sd">"""Pulls up to ``max_messages`` messages from Pub/Sub subscription.</span>
+
+<span class="sd"> :param project: the GCP project ID where the subscription exists</span>
+<span class="sd"> :type project: string</span>
+<span class="sd"> :param subscription: the Pub/Sub subscription name to pull from; do not</span>
+<span class="sd"> include the 'projects/{project}/topics/' prefix.</span>
+<span class="sd"> :type subscription: string</span>
+<span class="sd"> :param max_messages: The maximum number of messages to return from</span>
+<span class="sd"> the Pub/Sub API.</span>
+<span class="sd"> :type max_messages: int</span>
+<span class="sd"> :param return_immediately: If set, the Pub/Sub API will immediately</span>
+<span class="sd"> return if no messages are available. Otherwise, the request will</span>
+<span class="sd"> block for an undisclosed, but bounded period of time</span>
+<span class="sd"> :type return_immediately: bool</span>
+<span class="sd"> :return A list of Pub/Sub ReceivedMessage objects each containing</span>
+<span class="sd"> an ``ackId`` property and a ``message`` property, which includes</span>
+<span class="sd"> the base64-encoded message content. See</span>
+<span class="sd"> https://cloud.google.com/pubsub/docs/reference/rest/v1/\</span>
+<span class="sd"> projects.subscriptions/pull#ReceivedMessage</span>
+<span class="sd"> """</span>
+ <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">full_subscription</span> <span class="o">=</span> <span class="n">_format_subscription</span><span class="p">(</span><span class="n">project</span><span class="p">,</span> <span class="n">subscription</span><span class="p">)</span>
+ <span class="n">body</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'maxMessages'</span><span class="p">:</span> <span class="n">max_messages</span><span class="p">,</span>
+ <span class="s1">'returnImmediately'</span><span class="p">:</span> <span class="n">return_immediately</span>
+ <span class="p">}</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="n">service</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">subscriptions</span><span class="p">()</span><span class="o">.</span><span class="n">pull</span><span class="p">(</span>
+ <span class="n">subscription</span><span class="o">=</span><span class="n">full_subscription</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'receivedMessages'</span><span class="p">,</span> <span class="p">[])</span>
+ <span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">PubSubException</span><span class="p">(</span>
+ <span class="s1">'Error pulling messages from subscription </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">full_subscription</span><span class="p">),</span> <span class="n">e</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="PubSubHook.acknowledge"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.gcp_pubsub_hook.PubSubHook.acknowledge">[docs]</a> <span class="k">def</span> <span class="nf">acknowledge</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project</span><span class="p">,</span> <span class="n">subscription</span><span class="p">,</span> <span class="n">ack_ids</span><span class="p">):</span>
+ <span class="sd">"""Pulls up to ``max_messages`` messages from Pub/Sub subscription.</span>
+
+<span class="sd"> :param project: the GCP project name or ID in which to create</span>
+<span class="sd"> the topic</span>
+<span class="sd"> :type project: string</span>
+<span class="sd"> :param subscription: the Pub/Sub subscription name to delete; do not</span>
+<span class="sd"> include the 'projects/{project}/topics/' prefix.</span>
+<span class="sd"> :type subscription: string</span>
+<span class="sd"> :param ack_ids: List of ReceivedMessage ackIds from a previous pull</span>
+<span class="sd"> response</span>
+<span class="sd"> :type ack_ids: list</span>
+<span class="sd"> """</span>
+ <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">full_subscription</span> <span class="o">=</span> <span class="n">_format_subscription</span><span class="p">(</span><span class="n">project</span><span class="p">,</span> <span class="n">subscription</span><span class="p">)</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">service</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">subscriptions</span><span class="p">()</span><span class="o">.</span><span class="n">acknowledge</span><span class="p">(</span>
+ <span class="n">subscription</span><span class="o">=</span><span class="n">full_subscription</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="p">{</span><span class="s1">'ackIds'</span><span class="p">:</span> <span class="n">ack_ids</span><span class="p">}</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">PubSubException</span><span class="p">(</span>
+ <span class="s1">'Error acknowledging </span><span class="si">{}</span><span class="s1"> messages pulled from subscription </span><span class="si">{}</span><span class="s1">'</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">ack_ids</span><span class="p">),</span> <span class="n">full_subscription</span><span class="p">),</span> <span class="n">e</span><span class="p">)</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[42/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/pinot_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/pinot_hook.html b/_modules/airflow/contrib/hooks/pinot_hook.html
new file mode 100644
index 0000000..a34b554
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/pinot_hook.html
@@ -0,0 +1,340 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.pinot_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.pinot_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.pinot_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">six</span>
+
+<span class="kn">from</span> <span class="nn">pinotdb</span> <span class="k">import</span> <span class="n">connect</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="k">import</span> <span class="n">DbApiHook</span>
+
+
+<div class="viewcode-block" id="PinotDbApiHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.pinot_hook.PinotDbApiHook">[docs]</a><span class="k">class</span> <span class="nc">PinotDbApiHook</span><span class="p">(</span><span class="n">DbApiHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Connect to pinot db(https://github.com/linkedin/pinot) to issue pql</span>
+<span class="sd"> """</span>
+ <span class="n">conn_name_attr</span> <span class="o">=</span> <span class="s1">'pinot_broker_conn_id'</span>
+ <span class="n">default_conn_name</span> <span class="o">=</span> <span class="s1">'pinot_broker_default'</span>
+ <span class="n">supports_autocommit</span> <span class="o">=</span> <span class="kc">False</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">PinotDbApiHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+<div class="viewcode-block" id="PinotDbApiHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.pinot_hook.PinotDbApiHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Establish a connection to pinot broker through pinot dbqpi.</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">pinot_broker_conn_id</span><span class="p">)</span>
+ <span class="n">pinot_broker_conn</span> <span class="o">=</span> <span class="n">connect</span><span class="p">(</span>
+ <span class="n">host</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">,</span>
+ <span class="n">port</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">,</span>
+ <span class="n">path</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'endpoint'</span><span class="p">,</span> <span class="s1">'/pql'</span><span class="p">),</span>
+ <span class="n">scheme</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'schema'</span><span class="p">,</span> <span class="s1">'http'</span><span class="p">)</span>
+ <span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Get the connection to pinot '</span>
+ <span class="s1">'broker on </span><span class="si">{host}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">host</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">))</span>
+ <span class="k">return</span> <span class="n">pinot_broker_conn</span></div>
+
+<div class="viewcode-block" id="PinotDbApiHook.get_uri"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.pinot_hook.PinotDbApiHook.get_uri">[docs]</a> <span class="k">def</span> <span class="nf">get_uri</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Get the connection uri for pinot broker.</span>
+
+<span class="sd"> e.g: http://localhost:9000/pql</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_name_attr</span><span class="p">))</span>
+ <span class="n">host</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span>
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">host</span> <span class="o">+=</span> <span class="s1">':</span><span class="si">{port}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">port</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">)</span>
+ <span class="n">conn_type</span> <span class="o">=</span> <span class="s1">'http'</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">conn</span><span class="o">.</span><span class="n">conn_type</span> <span class="k">else</span> <span class="n">conn</span><span class="o">.</span><span class="n">conn_type</span>
+ <span class="n">endpoint</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'endpoint'</span><span class="p">,</span> <span class="s1">'pql'</span><span class="p">)</span>
+ <span class="k">return</span> <span class="s1">'</span><span class="si">{conn_type}</span><span class="s1">://</span><span class="si">{host}</span><span class="s1">/</span><span class="si">{endpoint}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">conn_type</span><span class="o">=</span><span class="n">conn_type</span><span class="p">,</span> <span class="n">host</span><span class="o">=</span><span class="n">host</span><span class="p">,</span> <span class="n">endpoint</span><span class="o">=</span><span class="n">endpoint</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="PinotDbApiHook.get_records"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.pinot_hook.PinotDbApiHook.get_records">[docs]</a> <span class="k">def</span> <span class="nf">get_records</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Executes the sql and returns a set of records.</span>
+
+<span class="sd"> :param sql: the sql statement to be executed (str) or a list of</span>
+<span class="sd"> sql statements to execute</span>
+<span class="sd"> :type sql: str</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">six</span><span class="o">.</span><span class="n">PY2</span><span class="p">:</span>
+ <span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span>
+
+ <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span> <span class="k">as</span> <span class="n">cur</span><span class="p">:</span>
+ <span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">cur</span><span class="o">.</span><span class="n">fetchall</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="PinotDbApiHook.get_first"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.pinot_hook.PinotDbApiHook.get_first">[docs]</a> <span class="k">def</span> <span class="nf">get_first</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Executes the sql and returns the first resulting row.</span>
+
+<span class="sd"> :param sql: the sql statement to be executed (str) or a list of</span>
+<span class="sd"> sql statements to execute</span>
+<span class="sd"> :type sql: str or list</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">six</span><span class="o">.</span><span class="n">PY2</span><span class="p">:</span>
+ <span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span>
+
+ <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span> <span class="k">as</span> <span class="n">cur</span><span class="p">:</span>
+ <span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">cur</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span></div>
+
+ <span class="k">def</span> <span class="nf">set_autocommit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn</span><span class="p">,</span> <span class="n">autocommit</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">()</span>
+
+ <span class="k">def</span> <span class="nf">get_pandas_df</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">()</span>
+
+ <span class="k">def</span> <span class="nf">insert_rows</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">rows</span><span class="p">,</span> <span class="n">target_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">commit_every</span><span class="o">=</span><span class="mi">1000</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">()</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/qubole_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/qubole_hook.html b/_modules/airflow/contrib/hooks/qubole_hook.html
new file mode 100644
index 0000000..c5fbc73
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/qubole_hook.html
@@ -0,0 +1,449 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.qubole_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.qubole_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.qubole_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+
+<span class="kn">import</span> <span class="nn">os</span>
+<span class="kn">import</span> <span class="nn">time</span>
+<span class="kn">import</span> <span class="nn">datetime</span>
+<span class="kn">import</span> <span class="nn">six</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">configuration</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span>
+
+<span class="kn">from</span> <span class="nn">qds_sdk.qubole</span> <span class="k">import</span> <span class="n">Qubole</span>
+<span class="kn">from</span> <span class="nn">qds_sdk.commands</span> <span class="k">import</span> <span class="n">Command</span><span class="p">,</span> <span class="n">HiveCommand</span><span class="p">,</span> <span class="n">PrestoCommand</span><span class="p">,</span> <span class="n">HadoopCommand</span><span class="p">,</span> \
+ <span class="n">PigCommand</span><span class="p">,</span> <span class="n">ShellCommand</span><span class="p">,</span> <span class="n">SparkCommand</span><span class="p">,</span> <span class="n">DbTapQueryCommand</span><span class="p">,</span> <span class="n">DbExportCommand</span><span class="p">,</span> \
+ <span class="n">DbImportCommand</span>
+
+
+<span class="n">COMMAND_CLASSES</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s2">"hivecmd"</span><span class="p">:</span> <span class="n">HiveCommand</span><span class="p">,</span>
+ <span class="s2">"prestocmd"</span><span class="p">:</span> <span class="n">PrestoCommand</span><span class="p">,</span>
+ <span class="s2">"hadoopcmd"</span><span class="p">:</span> <span class="n">HadoopCommand</span><span class="p">,</span>
+ <span class="s2">"shellcmd"</span><span class="p">:</span> <span class="n">ShellCommand</span><span class="p">,</span>
+ <span class="s2">"pigcmd"</span><span class="p">:</span> <span class="n">PigCommand</span><span class="p">,</span>
+ <span class="s2">"sparkcmd"</span><span class="p">:</span> <span class="n">SparkCommand</span><span class="p">,</span>
+ <span class="s2">"dbtapquerycmd"</span><span class="p">:</span> <span class="n">DbTapQueryCommand</span><span class="p">,</span>
+ <span class="s2">"dbexportcmd"</span><span class="p">:</span> <span class="n">DbExportCommand</span><span class="p">,</span>
+ <span class="s2">"dbimportcmd"</span><span class="p">:</span> <span class="n">DbImportCommand</span>
+<span class="p">}</span>
+
+<span class="n">HYPHEN_ARGS</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'cluster_label'</span><span class="p">,</span> <span class="s1">'app_id'</span><span class="p">,</span> <span class="s1">'note_id'</span><span class="p">]</span>
+
+<span class="n">POSITIONAL_ARGS</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'sub_command'</span><span class="p">,</span> <span class="s1">'parameters'</span><span class="p">]</span>
+
+<span class="n">COMMAND_ARGS</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s2">"hivecmd"</span><span class="p">:</span> <span class="p">[</span><span class="s1">'query'</span><span class="p">,</span> <span class="s1">'script_location'</span><span class="p">,</span> <span class="s1">'macros'</span><span class="p">,</span> <span class="s1">'tags'</span><span class="p">,</span> <span class="s1">'sample_size'</span><span class="p">,</span>
+ <span class="s1">'cluster_label'</span><span class="p">,</span> <span class="s1">'name'</span><span class="p">],</span>
+ <span class="s1">'prestocmd'</span><span class="p">:</span> <span class="p">[</span><span class="s1">'query'</span><span class="p">,</span> <span class="s1">'script_location'</span><span class="p">,</span> <span class="s1">'macros'</span><span class="p">,</span> <span class="s1">'tags'</span><span class="p">,</span> <span class="s1">'cluster_label'</span><span class="p">,</span> <span class="s1">'name'</span><span class="p">],</span>
+ <span class="s1">'hadoopcmd'</span><span class="p">:</span> <span class="p">[</span><span class="s1">'sub_command'</span><span class="p">,</span> <span class="s1">'tags'</span><span class="p">,</span> <span class="s1">'cluster_label'</span><span class="p">,</span> <span class="s1">'name'</span><span class="p">],</span>
+ <span class="s1">'shellcmd'</span><span class="p">:</span> <span class="p">[</span><span class="s1">'script'</span><span class="p">,</span> <span class="s1">'script_location'</span><span class="p">,</span> <span class="s1">'files'</span><span class="p">,</span> <span class="s1">'archives'</span><span class="p">,</span> <span class="s1">'parameters'</span><span class="p">,</span> <span class="s1">'tags'</span><span class="p">,</span>
+ <span class="s1">'cluster_label'</span><span class="p">,</span> <span class="s1">'name'</span><span class="p">],</span>
+ <span class="s1">'pigcmd'</span><span class="p">:</span> <span class="p">[</span><span class="s1">'script'</span><span class="p">,</span> <span class="s1">'script_location'</span><span class="p">,</span> <span class="s1">'parameters'</span><span class="p">,</span> <span class="s1">'tags'</span><span class="p">,</span> <span class="s1">'cluster_label'</span><span class="p">,</span>
+ <span class="s1">'name'</span><span class="p">],</span>
+ <span class="s1">'dbtapquerycmd'</span><span class="p">:</span> <span class="p">[</span><span class="s1">'db_tap_id'</span><span class="p">,</span> <span class="s1">'query'</span><span class="p">,</span> <span class="s1">'macros'</span><span class="p">,</span> <span class="s1">'tags'</span><span class="p">,</span> <span class="s1">'name'</span><span class="p">],</span>
+ <span class="s1">'sparkcmd'</span><span class="p">:</span> <span class="p">[</span><span class="s1">'program'</span><span class="p">,</span> <span class="s1">'cmdline'</span><span class="p">,</span> <span class="s1">'sql'</span><span class="p">,</span> <span class="s1">'script_location'</span><span class="p">,</span> <span class="s1">'macros'</span><span class="p">,</span> <span class="s1">'tags'</span><span class="p">,</span>
+ <span class="s1">'cluster_label'</span><span class="p">,</span> <span class="s1">'language'</span><span class="p">,</span> <span class="s1">'app_id'</span><span class="p">,</span> <span class="s1">'name'</span><span class="p">,</span> <span class="s1">'arguments'</span><span class="p">,</span> <span class="s1">'note_id'</span><span class="p">,</span>
+ <span class="s1">'user_program_arguments'</span><span class="p">],</span>
+ <span class="s1">'dbexportcmd'</span><span class="p">:</span> <span class="p">[</span><span class="s1">'mode'</span><span class="p">,</span> <span class="s1">'hive_table'</span><span class="p">,</span> <span class="s1">'partition_spec'</span><span class="p">,</span> <span class="s1">'dbtap_id'</span><span class="p">,</span> <span class="s1">'db_table'</span><span class="p">,</span>
+ <span class="s1">'db_update_mode'</span><span class="p">,</span> <span class="s1">'db_update_keys'</span><span class="p">,</span> <span class="s1">'export_dir'</span><span class="p">,</span>
+ <span class="s1">'fields_terminated_by'</span><span class="p">,</span> <span class="s1">'tags'</span><span class="p">,</span> <span class="s1">'name'</span><span class="p">],</span>
+ <span class="s1">'dbimportcmd'</span><span class="p">:</span> <span class="p">[</span><span class="s1">'mode'</span><span class="p">,</span> <span class="s1">'hive_table'</span><span class="p">,</span> <span class="s1">'dbtap_id'</span><span class="p">,</span> <span class="s1">'db_table'</span><span class="p">,</span> <span class="s1">'where_clause'</span><span class="p">,</span>
+ <span class="s1">'parallelism'</span><span class="p">,</span> <span class="s1">'extract_query'</span><span class="p">,</span> <span class="s1">'boundary_query'</span><span class="p">,</span> <span class="s1">'split_column'</span><span class="p">,</span>
+ <span class="s1">'tags'</span><span class="p">,</span> <span class="s1">'name'</span><span class="p">]</span>
+<span class="p">}</span>
+
+
+<div class="viewcode-block" id="QuboleHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.qubole_hook.QuboleHook">[docs]</a><span class="k">class</span> <span class="nc">QuboleHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">kwargs</span><span class="p">[</span><span class="s1">'qubole_conn_id'</span><span class="p">])</span>
+ <span class="n">Qubole</span><span class="o">.</span><span class="n">configure</span><span class="p">(</span><span class="n">api_token</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span> <span class="n">api_url</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">task_id</span> <span class="o">=</span> <span class="n">kwargs</span><span class="p">[</span><span class="s1">'task_id'</span><span class="p">]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">=</span> <span class="n">kwargs</span><span class="p">[</span><span class="s1">'dag'</span><span class="p">]</span><span class="o">.</span><span class="n">dag_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span> <span class="o">=</span> <span class="n">kwargs</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cls</span> <span class="o">=</span> <span class="n">COMMAND_CLASSES</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">[</span><span class="s1">'command_type'</span><span class="p">]]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cmd</span> <span class="o">=</span> <span class="kc">None</span>
+
+ <span class="nd">@staticmethod</span>
+ <span class="k">def</span> <span class="nf">handle_failure_retry</span><span class="p">(</span><span class="n">context</span><span class="p">):</span>
+ <span class="n">ti</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">'ti'</span><span class="p">]</span>
+ <span class="n">cmd_id</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">xcom_pull</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="s1">'qbol_cmd_id'</span><span class="p">,</span> <span class="n">task_ids</span><span class="o">=</span><span class="n">ti</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">cmd_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">=</span> <span class="n">Command</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="n">cmd_id</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">cmd</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">log</span> <span class="o">=</span> <span class="n">LoggingMixin</span><span class="p">()</span><span class="o">.</span><span class="n">log</span>
+ <span class="k">if</span> <span class="n">cmd</span><span class="o">.</span><span class="n">status</span> <span class="o">==</span> <span class="s1">'done'</span><span class="p">:</span>
+ <span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Command ID: </span><span class="si">%s</span><span class="s1"> has been succeeded, hence marking this '</span>
+ <span class="s1">'TI as Success.'</span><span class="p">,</span> <span class="n">cmd_id</span><span class="p">)</span>
+ <span class="n">ti</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span>
+ <span class="k">elif</span> <span class="n">cmd</span><span class="o">.</span><span class="n">status</span> <span class="o">==</span> <span class="s1">'running'</span><span class="p">:</span>
+ <span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Cancelling the Qubole Command Id: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">cmd_id</span><span class="p">)</span>
+ <span class="n">cmd</span><span class="o">.</span><span class="n">cancel</span><span class="p">()</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">args</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">create_cmd_args</span><span class="p">(</span><span class="n">context</span><span class="p">))</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="o">**</span><span class="n">args</span><span class="p">)</span>
+ <span class="n">context</span><span class="p">[</span><span class="s1">'task_instance'</span><span class="p">]</span><span class="o">.</span><span class="n">xcom_push</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="s1">'qbol_cmd_id'</span><span class="p">,</span> <span class="n">value</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cmd</span><span class="o">.</span><span class="n">id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s2">"Qubole command created with Id: </span><span class="si">%s</span><span class="s2"> and Status: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cmd</span><span class="o">.</span><span class="n">id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cmd</span><span class="o">.</span><span class="n">status</span>
+ <span class="p">)</span>
+
+ <span class="k">while</span> <span class="ow">not</span> <span class="n">Command</span><span class="o">.</span><span class="n">is_done</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cmd</span><span class="o">.</span><span class="n">status</span><span class="p">):</span>
+ <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="n">Qubole</span><span class="o">.</span><span class="n">poll_interval</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cmd</span><span class="o">.</span><span class="n">id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Command Id: </span><span class="si">%s</span><span class="s2"> and Status: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cmd</span><span class="o">.</span><span class="n">id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cmd</span><span class="o">.</span><span class="n">status</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="s1">'fetch_logs'</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">[</span><span class="s1">'fetch_logs'</span><span class="p">]</span> <span class="ow">is</span> <span class="kc">True</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Logs for Command Id: </span><span class="si">%s</span><span class="s2"> </span><span class="se">\n</span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cmd</span><span class="o">.</span><span class="n">id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cmd</span><span class="o">.</span><span class="n">get_log</span><span class="p">())</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cmd</span><span class="o">.</span><span class="n">status</span> <span class="o">!=</span> <span class="s1">'done'</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Command Id: </span><span class="si">{0}</span><span class="s1"> failed with Status: </span><span class="si">{1}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cmd</span><span class="o">.</span><span class="n">id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cmd</span><span class="o">.</span><span class="n">status</span><span class="p">))</span>
+
+<div class="viewcode-block" id="QuboleHook.kill"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.qubole_hook.QuboleHook.kill">[docs]</a> <span class="k">def</span> <span class="nf">kill</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ti</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Kill (cancel) a Qubole commmand</span>
+<span class="sd"> :param ti: Task Instance of the dag, used to determine the Quboles command id</span>
+<span class="sd"> :return: response from Qubole</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cmd</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">cmd_id</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">xcom_pull</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="s2">"qbol_cmd_id"</span><span class="p">,</span> <span class="n">task_ids</span><span class="o">=</span><span class="n">ti</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="n">cmd_id</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">cmd</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Sending KILL signal to Qubole Command Id: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cmd</span><span class="o">.</span><span class="n">id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cmd</span><span class="o">.</span><span class="n">cancel</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="QuboleHook.get_results"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.qubole_hook.QuboleHook.get_results">[docs]</a> <span class="k">def</span> <span class="nf">get_results</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ti</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">fp</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">inline</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">delim</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">fetch</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Get results (or just s3 locations) of a command from Qubole and save into a file</span>
+<span class="sd"> :param ti: Task Instance of the dag, used to determine the Quboles command id</span>
+<span class="sd"> :param fp: Optional file pointer, will create one and return if None passed</span>
+<span class="sd"> :param inline: True to download actual results, False to get s3 locations only</span>
+<span class="sd"> :param delim: Replaces the CTL-A chars with the given delim, defaults to ','</span>
+<span class="sd"> :param fetch: when inline is True, get results directly from s3 (if large)</span>
+<span class="sd"> :return: file location containing actual results or s3 locations of results</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">fp</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">iso</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
+ <span class="n">logpath</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">expanduser</span><span class="p">(</span>
+ <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'BASE_LOG_FOLDER'</span><span class="p">)</span>
+ <span class="p">)</span>
+ <span class="n">resultpath</span> <span class="o">=</span> <span class="n">logpath</span> <span class="o">+</span> <span class="s1">'/'</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">+</span> <span class="s1">'/'</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_id</span> <span class="o">+</span> <span class="s1">'/results'</span>
+ <span class="n">configuration</span><span class="o">.</span><span class="n">mkdir_p</span><span class="p">(</span><span class="n">resultpath</span><span class="p">)</span>
+ <span class="n">fp</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">resultpath</span> <span class="o">+</span> <span class="s1">'/'</span> <span class="o">+</span> <span class="n">iso</span><span class="p">,</span> <span class="s1">'wb'</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cmd</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">cmd_id</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">xcom_pull</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="s2">"qbol_cmd_id"</span><span class="p">,</span> <span class="n">task_ids</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="n">cmd_id</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">cmd</span><span class="o">.</span><span class="n">get_results</span><span class="p">(</span><span class="n">fp</span><span class="p">,</span> <span class="n">inline</span><span class="p">,</span> <span class="n">delim</span><span class="p">,</span> <span class="n">fetch</span><span class="p">)</span>
+ <span class="n">fp</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
+ <span class="n">fp</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">fp</span><span class="o">.</span><span class="n">name</span></div>
+
+<div class="viewcode-block" id="QuboleHook.get_log"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.qubole_hook.QuboleHook.get_log">[docs]</a> <span class="k">def</span> <span class="nf">get_log</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ti</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Get Logs of a command from Qubole</span>
+<span class="sd"> :param ti: Task Instance of the dag, used to determine the Quboles command id</span>
+<span class="sd"> :return: command log as text</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cmd</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">cmd_id</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">xcom_pull</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="s2">"qbol_cmd_id"</span><span class="p">,</span> <span class="n">task_ids</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span>
+ <span class="n">Command</span><span class="o">.</span><span class="n">get_log_id</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cls</span><span class="p">,</span> <span class="n">cmd_id</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="QuboleHook.get_jobs_id"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.qubole_hook.QuboleHook.get_jobs_id">[docs]</a> <span class="k">def</span> <span class="nf">get_jobs_id</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ti</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Get jobs associated with a Qubole commands</span>
+<span class="sd"> :param ti: Task Instance of the dag, used to determine the Quboles command id</span>
+<span class="sd"> :return: Job informations assoiciated with command</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cmd</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">cmd_id</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">xcom_pull</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="s2">"qbol_cmd_id"</span><span class="p">,</span> <span class="n">task_ids</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span>
+ <span class="n">Command</span><span class="o">.</span><span class="n">get_jobs_id</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cls</span><span class="p">,</span> <span class="n">cmd_id</span><span class="p">)</span></div>
+
+ <span class="k">def</span> <span class="nf">create_cmd_args</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">args</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="n">cmd_type</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">[</span><span class="s1">'command_type'</span><span class="p">]</span>
+ <span class="n">inplace_args</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="n">tags</span> <span class="o">=</span> <span class="nb">set</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="n">context</span><span class="p">[</span><span class="s1">'run_id'</span><span class="p">]])</span>
+
+ <span class="k">for</span> <span class="n">k</span><span class="p">,</span><span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+ <span class="k">if</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">COMMAND_ARGS</span><span class="p">[</span><span class="n">cmd_type</span><span class="p">]:</span>
+ <span class="k">if</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">HYPHEN_ARGS</span><span class="p">:</span>
+ <span class="n">args</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"--</span><span class="si">{0}</span><span class="s2">=</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">k</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'_'</span><span class="p">,</span> <span class="s1">'-'</span><span class="p">),</span><span class="n">v</span><span class="p">))</span>
+ <span class="k">elif</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">POSITIONAL_ARGS</span><span class="p">:</span>
+ <span class="n">inplace_args</span> <span class="o">=</span> <span class="n">v</span>
+ <span class="k">elif</span> <span class="n">k</span> <span class="o">==</span> <span class="s1">'tags'</span><span class="p">:</span>
+ <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">six</span><span class="o">.</span><span class="n">string_types</span><span class="p">):</span>
+ <span class="n">tags</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">v</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)):</span>
+ <span class="k">for</span> <span class="n">val</span> <span class="ow">in</span> <span class="n">v</span><span class="p">:</span>
+ <span class="n">tags</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">val</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">args</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"--</span><span class="si">{0}</span><span class="s2">=</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">k</span><span class="p">,</span><span class="n">v</span><span class="p">))</span>
+
+ <span class="k">if</span> <span class="n">k</span> <span class="o">==</span> <span class="s1">'notify'</span> <span class="ow">and</span> <span class="n">v</span> <span class="ow">is</span> <span class="kc">True</span><span class="p">:</span>
+ <span class="n">args</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"--notify"</span><span class="p">)</span>
+
+ <span class="n">args</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"--tags=</span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="s1">','</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="kc">None</span><span class="p">,</span><span class="n">tags</span><span class="p">))))</span>
+
+ <span class="k">if</span> <span class="n">inplace_args</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">args</span> <span class="o">+=</span> <span class="n">inplace_args</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">' '</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">args</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/redis_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/redis_hook.html b/_modules/airflow/contrib/hooks/redis_hook.html
new file mode 100644
index 0000000..1d0c202
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/redis_hook.html
@@ -0,0 +1,328 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.redis_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.redis_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.redis_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="sd">"""</span>
+<span class="sd">RedisHook module</span>
+<span class="sd">"""</span>
+<span class="kn">from</span> <span class="nn">redis</span> <span class="k">import</span> <span class="n">StrictRedis</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+
+<div class="viewcode-block" id="RedisHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.redis_hook.RedisHook">[docs]</a><span class="k">class</span> <span class="nc">RedisHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Hook to interact with Redis database</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">redis_conn_id</span><span class="o">=</span><span class="s1">'redis_default'</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Prepares hook to connect to a Redis database.</span>
+
+<span class="sd"> :param conn_id: the name of the connection that has the parameters</span>
+<span class="sd"> we need to connect to Redis.</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">redis_conn_id</span> <span class="o">=</span> <span class="n">redis_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">redis_conn_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">host</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">port</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">db</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'db'</span><span class="p">,</span> <span class="mi">0</span><span class="p">))</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
+ <span class="sd">'''Connection "{conn}":</span>
+<span class="sd"> \thost: {host}</span>
+<span class="sd"> \tport: {port}</span>
+<span class="sd"> \textra: {extra}</span>
+<span class="sd"> '''</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">conn</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">redis_conn_id</span><span class="p">,</span>
+ <span class="n">host</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">host</span><span class="p">,</span>
+ <span class="n">port</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">port</span><span class="p">,</span>
+ <span class="n">extra</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span>
+ <span class="p">)</span>
+ <span class="p">)</span>
+
+<div class="viewcode-block" id="RedisHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.redis_hook.RedisHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns a Redis connection.</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
+ <span class="s1">'generating Redis client for conn_id "</span><span class="si">%s</span><span class="s1">" on </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">redis_conn_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">port</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">db</span>
+ <span class="p">)</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="n">StrictRedis</span><span class="p">(</span>
+ <span class="n">host</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">host</span><span class="p">,</span>
+ <span class="n">port</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">port</span><span class="p">,</span>
+ <span class="n">password</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">password</span><span class="p">,</span>
+ <span class="n">db</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="p">)</span>
+ <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">general_error</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'Failed to create Redis client, error: </span><span class="si">{error}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">error</span><span class="o">=</span><span class="nb">str</span><span class="p">(</span><span class="n">general_error</span><span class="p">)</span>
+ <span class="p">)</span>
+ <span class="p">)</span>
+
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span></div>
+
+<div class="viewcode-block" id="RedisHook.key_exists"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.redis_hook.RedisHook.key_exists">[docs]</a> <span class="k">def</span> <span class="nf">key_exists</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Checks if a key exists in Redis database</span>
+
+<span class="sd"> :param key: The key to check the existence.</span>
+<span class="sd"> :type key: string</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">key</span><span class="p">)</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[19/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/sensors/redis_key_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/sensors/redis_key_sensor.html b/_modules/airflow/contrib/sensors/redis_key_sensor.html
new file mode 100644
index 0000000..55e26cd
--- /dev/null
+++ b/_modules/airflow/contrib/sensors/redis_key_sensor.html
@@ -0,0 +1,282 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.sensors.redis_key_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.sensors.redis_key_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.sensors.redis_key_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.redis_hook</span> <span class="k">import</span> <span class="n">RedisHook</span>
+<span class="kn">from</span> <span class="nn">airflow.sensors.base_sensor_operator</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="RedisKeySensor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.sensors.redis_key_sensor.RedisKeySensor">[docs]</a><span class="k">class</span> <span class="nc">RedisKeySensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Checks for the existence of a key in a Redis database</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'key'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f0eee4'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">redis_conn_id</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Create a new RedisKeySensor</span>
+
+<span class="sd"> :param key: The key to be monitored</span>
+<span class="sd"> :type key: string</span>
+<span class="sd"> :param redis_conn_id: The connection ID to use when connecting to Redis DB.</span>
+<span class="sd"> :type redis_conn_id: string</span>
+<span class="sd"> """</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">RedisKeySensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">redis_conn_id</span> <span class="o">=</span> <span class="n">redis_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">key</span> <span class="o">=</span> <span class="n">key</span>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Sensor check existence of key: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">key</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">RedisHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">redis_conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">key_exists</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">key</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/sensors/sftp_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/sensors/sftp_sensor.html b/_modules/airflow/contrib/sensors/sftp_sensor.html
new file mode 100644
index 0000000..1df8d4f
--- /dev/null
+++ b/_modules/airflow/contrib/sensors/sftp_sensor.html
@@ -0,0 +1,287 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.sensors.sftp_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.sensors.sftp_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.sensors.sftp_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+<span class="kn">from</span> <span class="nn">paramiko</span> <span class="k">import</span> <span class="n">SFTP_NO_SUCH_FILE</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.sftp_hook</span> <span class="k">import</span> <span class="n">SFTPHook</span>
+<span class="kn">from</span> <span class="nn">airflow.operators.sensors</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="SFTPSensor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.sensors.sftp_sensor.SFTPSensor">[docs]</a><span class="k">class</span> <span class="nc">SFTPSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Waits for a file or directory to be present on SFTP.</span>
+<span class="sd"> :param path: Remote file or directory path</span>
+<span class="sd"> :type path: str</span>
+<span class="sd"> :param sftp_conn_id: The connection to run the sensor against</span>
+<span class="sd"> :type sftp_conn_id: str</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'path'</span><span class="p">,)</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">sftp_conn_id</span><span class="o">=</span><span class="s1">'sftp_default'</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">SFTPSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">path</span> <span class="o">=</span> <span class="n">path</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">SFTPHook</span><span class="p">(</span><span class="n">sftp_conn_id</span><span class="o">=</span><span class="n">sftp_conn_id</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Poking for </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">path</span><span class="p">)</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">get_mod_time</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">path</span><span class="p">)</span>
+ <span class="k">except</span> <span class="ne">IOError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">e</span><span class="o">.</span><span class="n">errno</span> <span class="o">!=</span> <span class="n">SFTP_NO_SUCH_FILE</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">e</span>
+ <span class="k">return</span> <span class="kc">False</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">close_conn</span><span class="p">()</span>
+ <span class="k">return</span> <span class="kc">True</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/sensors/wasb_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/sensors/wasb_sensor.html b/_modules/airflow/contrib/sensors/wasb_sensor.html
index a1a8d95..8fdd499 100644
--- a/_modules/airflow/contrib/sensors/wasb_sensor.html
+++ b/_modules/airflow/contrib/sensors/wasb_sensor.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -169,20 +171,25 @@
<h1>Source code for airflow.contrib.sensors.wasb_sensor</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="c1">#</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.wasb_hook</span> <span class="k">import</span> <span class="n">WasbHook</span>
-<span class="kn">from</span> <span class="nn">airflow.operators.sensors</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.sensors.base_sensor_operator</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/executors/celery_executor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/executors/celery_executor.html b/_modules/airflow/executors/celery_executor.html
index 9601c12..b787b3e 100644
--- a/_modules/airflow/executors/celery_executor.html
+++ b/_modules/airflow/executors/celery_executor.html
@@ -13,6 +13,8 @@
+
+
@@ -30,6 +32,9 @@
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
<link rel="top" title="Airflow Documentation" href="../../../index.html"/>
<link rel="up" title="Module code" href="../../index.html"/>
@@ -40,6 +45,7 @@
<body class="wy-body-for-nav" role="document">
+
<div class="wy-grid-for-nav">
@@ -76,13 +82,16 @@
- <ul>
+
+
+
+ <ul>
<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
@@ -90,6 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
</ul>
@@ -104,8 +117,10 @@
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="../../../index.html">Airflow</a>
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
</nav>
@@ -118,19 +133,36 @@
+
+
+
+
+
+
+
+
+
+
<div role="navigation" aria-label="breadcrumbs navigation">
+
<ul class="wy-breadcrumbs">
- <li><a href="../../../index.html">Docs</a> »</li>
-
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
<li><a href="../../index.html">Module code</a> »</li>
-
- <li>airflow.executors.celery_executor</li>
+
+ <li>airflow.executors.celery_executor</li>
+
+
<li class="wy-breadcrumbs-aside">
-
+
</li>
+
</ul>
+
+
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
@@ -139,66 +171,72 @@
<h1>Source code for airflow.executors.celery_executor</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
-
-<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">object</span>
-<span class="kn">import</span> <span class="nn">logging</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
<span class="kn">import</span> <span class="nn">subprocess</span>
<span class="kn">import</span> <span class="nn">time</span>
+<span class="kn">import</span> <span class="nn">os</span>
-<span class="kn">from</span> <span class="nn">celery</span> <span class="kn">import</span> <span class="n">Celery</span>
-<span class="kn">from</span> <span class="nn">celery</span> <span class="kn">import</span> <span class="n">states</span> <span class="k">as</span> <span class="n">celery_states</span>
+<span class="kn">from</span> <span class="nn">celery</span> <span class="k">import</span> <span class="n">Celery</span>
+<span class="kn">from</span> <span class="nn">celery</span> <span class="k">import</span> <span class="n">states</span> <span class="k">as</span> <span class="n">celery_states</span>
-<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
-<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="kn">import</span> <span class="n">BaseExecutor</span>
-<span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">configuration</span>
+<span class="kn">from</span> <span class="nn">airflow.config_templates.default_celery</span> <span class="k">import</span> <span class="n">DEFAULT_CELERY_CONFIG</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="k">import</span> <span class="n">BaseExecutor</span>
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">configuration</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.module_loading</span> <span class="k">import</span> <span class="n">import_string</span>
-<span class="n">PARALLELISM</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'PARALLELISM'</span><span class="p">)</span>
+<span class="n">PARALLELISM</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'PARALLELISM'</span><span class="p">)</span>
<span class="sd">'''</span>
<span class="sd">To start the celery worker, run the command:</span>
<span class="sd">airflow worker</span>
<span class="sd">'''</span>
-<span class="n">DEFAULT_QUEUE</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'celery'</span><span class="p">,</span> <span class="s1">'DEFAULT_QUEUE'</span><span class="p">)</span>
-
-
-<span class="k">class</span> <span class="nc">CeleryConfig</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
- <span class="n">CELERY_ACCEPT_CONTENT</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'json'</span><span class="p">,</span> <span class="s1">'pickle'</span><span class="p">]</span>
- <span class="n">CELERYD_PREFETCH_MULTIPLIER</span> <span class="o">=</span> <span class="mi">1</span>
- <span class="n">CELERY_ACKS_LATE</span> <span class="o">=</span> <span class="bp">True</span>
- <span class="n">BROKER_URL</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'celery'</span><span class="p">,</span> <span class="s1">'BROKER_URL'</span><span class="p">)</span>
- <span class="n">CELERY_RESULT_BACKEND</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'celery'</span><span class="p">,</span> <span class="s1">'CELERY_RESULT_BACKEND'</span><span class="p">)</span>
- <span class="n">CELERYD_CONCURRENCY</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'celery'</span><span class="p">,</span> <span class="s1">'CELERYD_CONCURRENCY'</span><span class="p">)</span>
- <span class="n">CELERY_DEFAULT_QUEUE</span> <span class="o">=</span> <span class="n">DEFAULT_QUEUE</span>
- <span class="n">CELERY_DEFAULT_EXCHANGE</span> <span class="o">=</span> <span class="n">DEFAULT_QUEUE</span>
+<span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">has_option</span><span class="p">(</span><span class="s1">'celery'</span><span class="p">,</span> <span class="s1">'celery_config_options'</span><span class="p">):</span>
+ <span class="n">celery_configuration</span> <span class="o">=</span> <span class="n">import_string</span><span class="p">(</span>
+ <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'celery'</span><span class="p">,</span> <span class="s1">'celery_config_options'</span><span class="p">)</span>
+ <span class="p">)</span>
+<span class="k">else</span><span class="p">:</span>
+ <span class="n">celery_configuration</span> <span class="o">=</span> <span class="n">DEFAULT_CELERY_CONFIG</span>
<span class="n">app</span> <span class="o">=</span> <span class="n">Celery</span><span class="p">(</span>
- <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'celery'</span><span class="p">,</span> <span class="s1">'CELERY_APP_NAME'</span><span class="p">),</span>
- <span class="n">config_source</span><span class="o">=</span><span class="n">CeleryConfig</span><span class="p">)</span>
+ <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'celery'</span><span class="p">,</span> <span class="s1">'CELERY_APP_NAME'</span><span class="p">),</span>
+ <span class="n">config_source</span><span class="o">=</span><span class="n">celery_configuration</span><span class="p">)</span>
-<span class="nd">@app.task</span>
+<span class="nd">@app</span><span class="o">.</span><span class="n">task</span>
<span class="k">def</span> <span class="nf">execute_command</span><span class="p">(</span><span class="n">command</span><span class="p">):</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Executing command in Celery "</span> <span class="o">+</span> <span class="n">command</span><span class="p">)</span>
+ <span class="n">log</span> <span class="o">=</span> <span class="n">LoggingMixin</span><span class="p">()</span><span class="o">.</span><span class="n">log</span>
+ <span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Executing command in Celery: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">command</span><span class="p">)</span>
+ <span class="n">env</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="k">try</span><span class="p">:</span>
- <span class="n">subprocess</span><span class="o">.</span><span class="n">check_call</span><span class="p">(</span><span class="n">command</span><span class="p">,</span> <span class="n">shell</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
+ <span class="n">subprocess</span><span class="o">.</span><span class="n">check_call</span><span class="p">(</span><span class="n">command</span><span class="p">,</span> <span class="n">shell</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">stderr</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">STDOUT</span><span class="p">,</span>
+ <span class="n">close_fds</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">env</span><span class="o">=</span><span class="n">env</span><span class="p">)</span>
<span class="k">except</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">CalledProcessError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
+ <span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s1">'execute_command encountered a CalledProcessError'</span><span class="p">)</span>
+ <span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">output</span><span class="p">)</span>
+
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Celery command failed'</span><span class="p">)</span>
-<div class="viewcode-block" id="CeleryExecutor"><a class="viewcode-back" href="../../../code.html#airflow.executors.CeleryExecutor">[docs]</a><span class="k">class</span> <span class="nc">CeleryExecutor</span><span class="p">(</span><span class="n">BaseExecutor</span><span class="p">):</span>
+<div class="viewcode-block" id="CeleryExecutor"><a class="viewcode-back" href="../../../code.html#airflow.executors.celery_executor.CeleryExecutor">[docs]</a><span class="k">class</span> <span class="nc">CeleryExecutor</span><span class="p">(</span><span class="n">BaseExecutor</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> CeleryExecutor is recommended for production use of Airflow. It allows</span>
<span class="sd"> distributing the execution of task instances to multiple worker nodes.</span>
@@ -207,51 +245,57 @@
<span class="sd"> vast amounts of messages, while providing operations with the tools</span>
<span class="sd"> required to maintain such a system.</span>
<span class="sd"> """</span>
-
<span class="k">def</span> <span class="nf">start</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">tasks</span> <span class="o">=</span> <span class="p">{}</span>
<span class="bp">self</span><span class="o">.</span><span class="n">last_state</span> <span class="o">=</span> <span class="p">{}</span>
- <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="n">DEFAULT_QUEUE</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span> <span class="s2">"[celery] queuing {key} through celery, "</span>
- <span class="s2">"queue={queue}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
+ <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span>
+ <span class="n">queue</span><span class="o">=</span><span class="n">DEFAULT_CELERY_CONFIG</span><span class="p">[</span><span class="s1">'task_default_queue'</span><span class="p">],</span>
+ <span class="n">executor_config</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span> <span class="s2">"[celery] queuing </span><span class="si">{key}</span><span class="s2"> through celery, "</span>
+ <span class="s2">"queue=</span><span class="si">{queue}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
<span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">execute_command</span><span class="o">.</span><span class="n">apply_async</span><span class="p">(</span>
<span class="n">args</span><span class="o">=</span><span class="p">[</span><span class="n">command</span><span class="p">],</span> <span class="n">queue</span><span class="o">=</span><span class="n">queue</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">last_state</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">celery_states</span><span class="o">.</span><span class="n">PENDING</span>
<span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-
- <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
- <span class="s2">"Inquiring about {} celery task(s)"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">)))</span>
- <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">async</span> <span class="ow">in</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="o">.</span><span class="n">items</span><span class="p">()):</span>
- <span class="n">state</span> <span class="o">=</span> <span class="n">async</span><span class="o">.</span><span class="n">state</span>
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_state</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">!=</span> <span class="n">state</span><span class="p">:</span>
- <span class="k">if</span> <span class="n">state</span> <span class="o">==</span> <span class="n">celery_states</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">success</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
- <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
- <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_state</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
- <span class="k">elif</span> <span class="n">state</span> <span class="o">==</span> <span class="n">celery_states</span><span class="o">.</span><span class="n">FAILURE</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">fail</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
- <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
- <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_state</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
- <span class="k">elif</span> <span class="n">state</span> <span class="o">==</span> <span class="n">celery_states</span><span class="o">.</span><span class="n">REVOKED</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">fail</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
- <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
- <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_state</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Unexpected state: "</span> <span class="o">+</span> <span class="n">async</span><span class="o">.</span><span class="n">state</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">last_state</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">async</span><span class="o">.</span><span class="n">state</span>
-
- <span class="k">def</span> <span class="nf">end</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">synchronous</span><span class="o">=</span><span class="bp">False</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Inquiring about </span><span class="si">%s</span><span class="s2"> celery task(s)"</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">))</span>
+ <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="k">async</span> <span class="ow">in</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="o">.</span><span class="n">items</span><span class="p">()):</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">state</span> <span class="o">=</span> <span class="k">async</span><span class="o">.</span><span class="n">state</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_state</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">!=</span> <span class="n">state</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">state</span> <span class="o">==</span> <span class="n">celery_states</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">success</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
+ <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
+ <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_state</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
+ <span class="k">elif</span> <span class="n">state</span> <span class="o">==</span> <span class="n">celery_states</span><span class="o">.</span><span class="n">FAILURE</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">fail</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
+ <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
+ <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_state</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
+ <span class="k">elif</span> <span class="n">state</span> <span class="o">==</span> <span class="n">celery_states</span><span class="o">.</span><span class="n">REVOKED</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">fail</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
+ <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
+ <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_state</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Unexpected state: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="k">async</span><span class="o">.</span><span class="n">state</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">last_state</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="k">async</span><span class="o">.</span><span class="n">state</span>
+ <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Error syncing the celery executor, ignoring it:"</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">end</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">synchronous</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
<span class="k">if</span> <span class="n">synchronous</span><span class="p">:</span>
<span class="k">while</span> <span class="nb">any</span><span class="p">([</span>
- <span class="n">async</span><span class="o">.</span><span class="n">state</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">celery_states</span><span class="o">.</span><span class="n">READY_STATES</span>
- <span class="k">for</span> <span class="n">async</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="o">.</span><span class="n">values</span><span class="p">()]):</span>
+ <span class="k">async</span><span class="o">.</span><span class="n">state</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">celery_states</span><span class="o">.</span><span class="n">READY_STATES</span>
+ <span class="k">for</span> <span class="k">async</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="o">.</span><span class="n">values</span><span class="p">()]):</span>
<span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">sync</span><span class="p">()</span></div>
</pre></div>
</div>
+ <div class="articleComments">
+
+ </div>
</div>
<footer>
@@ -284,7 +328,8 @@
VERSION:'',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
- HAS_SOURCE: true
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="../../../_static/jquery.js"></script>
[23/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/sqoop_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/sqoop_operator.html b/_modules/airflow/contrib/operators/sqoop_operator.html
new file mode 100644
index 0000000..31fafec
--- /dev/null
+++ b/_modules/airflow/contrib/operators/sqoop_operator.html
@@ -0,0 +1,467 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.sqoop_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.sqoop_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.sqoop_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+
+<span class="sd">"""</span>
+<span class="sd">This module contains a sqoop 1 operator</span>
+<span class="sd">"""</span>
+<span class="kn">import</span> <span class="nn">os</span>
+<span class="kn">import</span> <span class="nn">signal</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.sqoop_hook</span> <span class="k">import</span> <span class="n">SqoopHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="SqoopOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.sqoop_operator.SqoopOperator">[docs]</a><span class="k">class</span> <span class="nc">SqoopOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Execute a Sqoop job.</span>
+<span class="sd"> Documentation for Apache Sqoop can be found here: https://sqoop.apache.org/docs/1.4.2/SqoopUserGuide.html.</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'conn_id'</span><span class="p">,</span> <span class="s1">'cmd_type'</span><span class="p">,</span> <span class="s1">'table'</span><span class="p">,</span> <span class="s1">'query'</span><span class="p">,</span> <span class="s1">'target_dir'</span><span class="p">,</span> <span class="s1">'file_type'</span><span class="p">,</span> <span class="s1">'columns'</span><span class="p">,</span> <span class="s1">'split_by'</span><span class="p">,</span>
+ <span class="s1">'where'</span><span class="p">,</span> <span class="s1">'export_dir'</span><span class="p">,</span> <span class="s1">'input_null_string'</span><span class="p">,</span> <span class="s1">'input_null_non_string'</span><span class="p">,</span> <span class="s1">'staging_table'</span><span class="p">,</span>
+ <span class="s1">'enclosed_by'</span><span class="p">,</span> <span class="s1">'escaped_by'</span><span class="p">,</span> <span class="s1">'input_fields_terminated_by'</span><span class="p">,</span> <span class="s1">'input_lines_terminated_by'</span><span class="p">,</span>
+ <span class="s1">'input_optionally_enclosed_by'</span><span class="p">,</span> <span class="s1">'properties'</span><span class="p">,</span> <span class="s1">'extra_import_options'</span><span class="p">,</span> <span class="s1">'driver'</span><span class="p">,</span>
+ <span class="s1">'extra_export_options'</span><span class="p">,</span> <span class="s1">'hcatalog_database'</span><span class="p">,</span> <span class="s1">'hcatalog_table'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#7D8CA4'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">conn_id</span><span class="o">=</span><span class="s1">'sqoop_default'</span><span class="p">,</span>
+ <span class="n">cmd_type</span><span class="o">=</span><span class="s1">'import'</span><span class="p">,</span>
+ <span class="n">table</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">query</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">target_dir</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">append</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">file_type</span><span class="o">=</span><span class="s1">'text'</span><span class="p">,</span>
+ <span class="n">columns</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">num_mappers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">split_by</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">where</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">export_dir</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">input_null_string</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">input_null_non_string</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">staging_table</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">clear_staging_table</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">enclosed_by</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">escaped_by</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">input_fields_terminated_by</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">input_lines_terminated_by</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">input_optionally_enclosed_by</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">batch</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">direct</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">driver</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">relaxed_isolation</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">properties</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">hcatalog_database</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">hcatalog_table</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">create_hcatalog_table</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">extra_import_options</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">extra_export_options</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> :param conn_id: str</span>
+<span class="sd"> :param cmd_type: str specify command to execute "export" or "import"</span>
+<span class="sd"> :param table: Table to read</span>
+<span class="sd"> :param query: Import result of arbitrary SQL query. Instead of using the table,</span>
+<span class="sd"> columns and where arguments, you can specify a SQL statement with the query</span>
+<span class="sd"> argument. Must also specify a destination directory with target_dir.</span>
+<span class="sd"> :param target_dir: HDFS destination directory where the data</span>
+<span class="sd"> from the rdbms will be written</span>
+<span class="sd"> :param append: Append data to an existing dataset in HDFS</span>
+<span class="sd"> :param file_type: "avro", "sequence", "text" Imports data to</span>
+<span class="sd"> into the specified format. Defaults to text.</span>
+<span class="sd"> :param columns: <col,col,col> Columns to import from table</span>
+<span class="sd"> :param num_mappers: Use n mapper tasks to import/export in parallel</span>
+<span class="sd"> :param split_by: Column of the table used to split work units</span>
+<span class="sd"> :param where: WHERE clause to use during import</span>
+<span class="sd"> :param export_dir: HDFS Hive database directory to export to the rdbms</span>
+<span class="sd"> :param input_null_string: The string to be interpreted as null</span>
+<span class="sd"> for string columns</span>
+<span class="sd"> :param input_null_non_string: The string to be interpreted as null</span>
+<span class="sd"> for non-string columns</span>
+<span class="sd"> :param staging_table: The table in which data will be staged before</span>
+<span class="sd"> being inserted into the destination table</span>
+<span class="sd"> :param clear_staging_table: Indicate that any data present in the</span>
+<span class="sd"> staging table can be deleted</span>
+<span class="sd"> :param enclosed_by: Sets a required field enclosing character</span>
+<span class="sd"> :param escaped_by: Sets the escape character</span>
+<span class="sd"> :param input_fields_terminated_by: Sets the input field separator</span>
+<span class="sd"> :param input_lines_terminated_by: Sets the input end-of-line character</span>
+<span class="sd"> :param input_optionally_enclosed_by: Sets a field enclosing character</span>
+<span class="sd"> :param batch: Use batch mode for underlying statement execution</span>
+<span class="sd"> :param direct: Use direct export fast path</span>
+<span class="sd"> :param driver: Manually specify JDBC driver class to use</span>
+<span class="sd"> :param verbose: Switch to more verbose logging for debug purposes</span>
+<span class="sd"> :param relaxed_isolation: use read uncommitted isolation level</span>
+<span class="sd"> :param hcatalog_database: Specifies the database name for the HCatalog table</span>
+<span class="sd"> :param hcatalog_table: The argument value for this option is the HCatalog table</span>
+<span class="sd"> :param create_hcatalog_table: Have sqoop create the hcatalog table passed in or not</span>
+<span class="sd"> :param properties: additional JVM properties passed to sqoop</span>
+<span class="sd"> :param extra_import_options: Extra import options to pass as dict.</span>
+<span class="sd"> If a key doesn't have a value, just pass an empty string to it.</span>
+<span class="sd"> Don't include prefix of -- for sqoop options.</span>
+<span class="sd"> :param extra_export_options: Extra export options to pass as dict.</span>
+<span class="sd"> If a key doesn't have a value, just pass an empty string to it.</span>
+<span class="sd"> Don't include prefix of -- for sqoop options.</span>
+<span class="sd"> """</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">SqoopOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span> <span class="o">=</span> <span class="n">conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cmd_type</span> <span class="o">=</span> <span class="n">cmd_type</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">table</span> <span class="o">=</span> <span class="n">table</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">query</span> <span class="o">=</span> <span class="n">query</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">target_dir</span> <span class="o">=</span> <span class="n">target_dir</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">append</span> <span class="o">=</span> <span class="n">append</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">file_type</span> <span class="o">=</span> <span class="n">file_type</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">columns</span> <span class="o">=</span> <span class="n">columns</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">num_mappers</span> <span class="o">=</span> <span class="n">num_mappers</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">split_by</span> <span class="o">=</span> <span class="n">split_by</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">where</span> <span class="o">=</span> <span class="n">where</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">export_dir</span> <span class="o">=</span> <span class="n">export_dir</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">input_null_string</span> <span class="o">=</span> <span class="n">input_null_string</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">input_null_non_string</span> <span class="o">=</span> <span class="n">input_null_non_string</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">staging_table</span> <span class="o">=</span> <span class="n">staging_table</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">clear_staging_table</span> <span class="o">=</span> <span class="n">clear_staging_table</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">enclosed_by</span> <span class="o">=</span> <span class="n">enclosed_by</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">escaped_by</span> <span class="o">=</span> <span class="n">escaped_by</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">input_fields_terminated_by</span> <span class="o">=</span> <span class="n">input_fields_terminated_by</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">input_lines_terminated_by</span> <span class="o">=</span> <span class="n">input_lines_terminated_by</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">input_optionally_enclosed_by</span> <span class="o">=</span> <span class="n">input_optionally_enclosed_by</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">batch</span> <span class="o">=</span> <span class="n">batch</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">direct</span> <span class="o">=</span> <span class="n">direct</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">driver</span> <span class="o">=</span> <span class="n">driver</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">relaxed_isolation</span> <span class="o">=</span> <span class="n">relaxed_isolation</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hcatalog_database</span> <span class="o">=</span> <span class="n">hcatalog_database</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hcatalog_table</span> <span class="o">=</span> <span class="n">hcatalog_table</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">create_hcatalog_table</span> <span class="o">=</span> <span class="n">create_hcatalog_table</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">properties</span> <span class="o">=</span> <span class="n">properties</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">extra_import_options</span> <span class="o">=</span> <span class="n">extra_import_options</span> <span class="ow">or</span> <span class="p">{}</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">extra_export_options</span> <span class="o">=</span> <span class="n">extra_export_options</span> <span class="ow">or</span> <span class="p">{}</span>
+
+<div class="viewcode-block" id="SqoopOperator.execute"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.sqoop_operator.SqoopOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Execute sqoop job</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">SqoopHook</span><span class="p">(</span>
+ <span class="n">conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">,</span>
+ <span class="n">verbose</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">,</span>
+ <span class="n">num_mappers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_mappers</span><span class="p">,</span>
+ <span class="n">hcatalog_database</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hcatalog_database</span><span class="p">,</span>
+ <span class="n">hcatalog_table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hcatalog_table</span><span class="p">,</span>
+ <span class="n">properties</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">properties</span>
+ <span class="p">)</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cmd_type</span> <span class="o">==</span> <span class="s1">'export'</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">export_table</span><span class="p">(</span>
+ <span class="n">table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="p">,</span>
+ <span class="n">export_dir</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">export_dir</span><span class="p">,</span>
+ <span class="n">input_null_string</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">input_null_string</span><span class="p">,</span>
+ <span class="n">input_null_non_string</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">input_null_non_string</span><span class="p">,</span>
+ <span class="n">staging_table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">staging_table</span><span class="p">,</span>
+ <span class="n">clear_staging_table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">clear_staging_table</span><span class="p">,</span>
+ <span class="n">enclosed_by</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">enclosed_by</span><span class="p">,</span>
+ <span class="n">escaped_by</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">escaped_by</span><span class="p">,</span>
+ <span class="n">input_fields_terminated_by</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">input_fields_terminated_by</span><span class="p">,</span>
+ <span class="n">input_lines_terminated_by</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">input_lines_terminated_by</span><span class="p">,</span>
+ <span class="n">input_optionally_enclosed_by</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">input_optionally_enclosed_by</span><span class="p">,</span>
+ <span class="n">batch</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">batch</span><span class="p">,</span>
+ <span class="n">relaxed_isolation</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">relaxed_isolation</span><span class="p">,</span>
+ <span class="n">extra_export_options</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">extra_export_options</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">cmd_type</span> <span class="o">==</span> <span class="s1">'import'</span><span class="p">:</span>
+ <span class="c1"># add create hcatalog table to extra import options if option passed</span>
+ <span class="c1"># if new params are added to constructor can pass them in here so don't modify sqoop_hook for each param</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">create_hcatalog_table</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">extra_import_options</span><span class="p">[</span><span class="s1">'create-hcatalog-table'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">''</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">query</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'Cannot specify query and table together. Need to specify either or.'</span>
+ <span class="p">)</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">import_table</span><span class="p">(</span>
+ <span class="n">table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="p">,</span>
+ <span class="n">target_dir</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">target_dir</span><span class="p">,</span>
+ <span class="n">append</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">append</span><span class="p">,</span>
+ <span class="n">file_type</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">file_type</span><span class="p">,</span>
+ <span class="n">columns</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">columns</span><span class="p">,</span>
+ <span class="n">split_by</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">split_by</span><span class="p">,</span>
+ <span class="n">where</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">where</span><span class="p">,</span>
+ <span class="n">direct</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">direct</span><span class="p">,</span>
+ <span class="n">driver</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">driver</span><span class="p">,</span>
+ <span class="n">extra_import_options</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">extra_import_options</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">query</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">import_query</span><span class="p">(</span>
+ <span class="n">query</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">query</span><span class="p">,</span>
+ <span class="n">target_dir</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">target_dir</span><span class="p">,</span>
+ <span class="n">append</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">append</span><span class="p">,</span>
+ <span class="n">file_type</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">file_type</span><span class="p">,</span>
+ <span class="n">split_by</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">split_by</span><span class="p">,</span>
+ <span class="n">direct</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">direct</span><span class="p">,</span>
+ <span class="n">driver</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">driver</span><span class="p">,</span>
+ <span class="n">extra_import_options</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">extra_import_options</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s2">"Provide query or table parameter to import using Sqoop"</span>
+ <span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"cmd_type should be 'import' or 'export'"</span><span class="p">)</span></div>
+
+ <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Sending SIGTERM signal to bash process group'</span><span class="p">)</span>
+ <span class="n">os</span><span class="o">.</span><span class="n">killpg</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">getpgid</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">sp</span><span class="o">.</span><span class="n">pid</span><span class="p">),</span> <span class="n">signal</span><span class="o">.</span><span class="n">SIGTERM</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/ssh_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/ssh_operator.html b/_modules/airflow/contrib/operators/ssh_operator.html
new file mode 100644
index 0000000..d971825
--- /dev/null
+++ b/_modules/airflow/contrib/operators/ssh_operator.html
@@ -0,0 +1,394 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.ssh_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.ssh_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.ssh_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">base64</span> <span class="k">import</span> <span class="n">b64encode</span>
+<span class="kn">from</span> <span class="nn">select</span> <span class="k">import</span> <span class="n">select</span>
+
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">configuration</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.ssh_hook</span> <span class="k">import</span> <span class="n">SSHHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="SSHOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.ssh_operator.SSHOperator">[docs]</a><span class="k">class</span> <span class="nc">SSHOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> SSHOperator to execute commands on given remote host using the ssh_hook.</span>
+
+<span class="sd"> :param ssh_hook: predefined ssh_hook to use for remote execution</span>
+<span class="sd"> :type ssh_hook: :class:`SSHHook`</span>
+<span class="sd"> :param ssh_conn_id: connection id from airflow Connections</span>
+<span class="sd"> :type ssh_conn_id: str</span>
+<span class="sd"> :param remote_host: remote host to connect</span>
+<span class="sd"> :type remote_host: str</span>
+<span class="sd"> :param command: command to execute on remote host</span>
+<span class="sd"> :type command: str</span>
+<span class="sd"> :param timeout: timeout (in seconds) for executing the command.</span>
+<span class="sd"> :type timeout: int</span>
+<span class="sd"> :param do_xcom_push: return the stdout which also get set in xcom by airflow platform</span>
+<span class="sd"> :type do_xcom_push: bool</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'command'</span><span class="p">,)</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sh'</span><span class="p">,)</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">ssh_hook</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">ssh_conn_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">remote_host</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">command</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">timeout</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span>
+ <span class="n">do_xcom_push</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">SSHOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">ssh_hook</span> <span class="o">=</span> <span class="n">ssh_hook</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span> <span class="o">=</span> <span class="n">ssh_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span> <span class="o">=</span> <span class="n">remote_host</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">command</span> <span class="o">=</span> <span class="n">command</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span> <span class="o">=</span> <span class="n">timeout</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">do_xcom_push</span> <span class="o">=</span> <span class="n">do_xcom_push</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_hook</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">ssh_hook</span> <span class="o">=</span> <span class="n">SSHHook</span><span class="p">(</span><span class="n">ssh_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_hook</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"can not operate without ssh_hook or ssh_conn_id"</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">ssh_hook</span><span class="o">.</span><span class="n">remote_host</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span>
+
+ <span class="n">ssh_client</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">command</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"no command specified so nothing to execute here."</span><span class="p">)</span>
+
+ <span class="c1"># Auto apply tty when its required in case of sudo</span>
+ <span class="n">get_pty</span> <span class="o">=</span> <span class="kc">False</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">command</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'sudo'</span><span class="p">):</span>
+ <span class="n">get_pty</span> <span class="o">=</span> <span class="kc">True</span>
+
+ <span class="c1"># set timeout taken as params</span>
+ <span class="n">stdin</span><span class="p">,</span> <span class="n">stdout</span><span class="p">,</span> <span class="n">stderr</span> <span class="o">=</span> <span class="n">ssh_client</span><span class="o">.</span><span class="n">exec_command</span><span class="p">(</span><span class="n">command</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">command</span><span class="p">,</span>
+ <span class="n">get_pty</span><span class="o">=</span><span class="n">get_pty</span><span class="p">,</span>
+ <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span>
+ <span class="p">)</span>
+ <span class="c1"># get channels</span>
+ <span class="n">channel</span> <span class="o">=</span> <span class="n">stdout</span><span class="o">.</span><span class="n">channel</span>
+
+ <span class="c1"># closing stdin</span>
+ <span class="n">stdin</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
+ <span class="n">channel</span><span class="o">.</span><span class="n">shutdown_write</span><span class="p">()</span>
+
+ <span class="n">agg_stdout</span> <span class="o">=</span> <span class="sa">b</span><span class="s1">''</span>
+ <span class="n">agg_stderr</span> <span class="o">=</span> <span class="sa">b</span><span class="s1">''</span>
+
+ <span class="c1"># capture any initial output in case channel is closed already</span>
+ <span class="n">stdout_buffer_length</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">stdout</span><span class="o">.</span><span class="n">channel</span><span class="o">.</span><span class="n">in_buffer</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">stdout_buffer_length</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
+ <span class="n">agg_stdout</span> <span class="o">+=</span> <span class="n">stdout</span><span class="o">.</span><span class="n">channel</span><span class="o">.</span><span class="n">recv</span><span class="p">(</span><span class="n">stdout_buffer_length</span><span class="p">)</span>
+
+ <span class="c1"># read from both stdout and stderr</span>
+ <span class="k">while</span> <span class="ow">not</span> <span class="n">channel</span><span class="o">.</span><span class="n">closed</span> <span class="ow">or</span> <span class="n">channel</span><span class="o">.</span><span class="n">recv_ready</span><span class="p">()</span> <span class="ow">or</span> <span class="n">channel</span><span class="o">.</span><span class="n">recv_stderr_ready</span><span class="p">():</span>
+ <span class="n">readq</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">select</span><span class="p">([</span><span class="n">channel</span><span class="p">],</span> <span class="p">[],</span> <span class="p">[],</span> <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span>
+ <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">readq</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">c</span><span class="o">.</span><span class="n">recv_ready</span><span class="p">():</span>
+ <span class="n">line</span> <span class="o">=</span> <span class="n">stdout</span><span class="o">.</span><span class="n">channel</span><span class="o">.</span><span class="n">recv</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">in_buffer</span><span class="p">))</span>
+ <span class="n">line</span> <span class="o">=</span> <span class="n">line</span>
+ <span class="n">agg_stdout</span> <span class="o">+=</span> <span class="n">line</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">line</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">))</span>
+ <span class="k">if</span> <span class="n">c</span><span class="o">.</span><span class="n">recv_stderr_ready</span><span class="p">():</span>
+ <span class="n">line</span> <span class="o">=</span> <span class="n">stderr</span><span class="o">.</span><span class="n">channel</span><span class="o">.</span><span class="n">recv_stderr</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">in_stderr_buffer</span><span class="p">))</span>
+ <span class="n">line</span> <span class="o">=</span> <span class="n">line</span>
+ <span class="n">agg_stderr</span> <span class="o">+=</span> <span class="n">line</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">line</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">))</span>
+ <span class="k">if</span> <span class="n">stdout</span><span class="o">.</span><span class="n">channel</span><span class="o">.</span><span class="n">exit_status_ready</span><span class="p">()</span>\
+ <span class="ow">and</span> <span class="ow">not</span> <span class="n">stderr</span><span class="o">.</span><span class="n">channel</span><span class="o">.</span><span class="n">recv_stderr_ready</span><span class="p">()</span>\
+ <span class="ow">and</span> <span class="ow">not</span> <span class="n">stdout</span><span class="o">.</span><span class="n">channel</span><span class="o">.</span><span class="n">recv_ready</span><span class="p">():</span>
+ <span class="n">stdout</span><span class="o">.</span><span class="n">channel</span><span class="o">.</span><span class="n">shutdown_read</span><span class="p">()</span>
+ <span class="n">stdout</span><span class="o">.</span><span class="n">channel</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
+ <span class="k">break</span>
+
+ <span class="n">stdout</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
+ <span class="n">stderr</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
+
+ <span class="n">exit_status</span> <span class="o">=</span> <span class="n">stdout</span><span class="o">.</span><span class="n">channel</span><span class="o">.</span><span class="n">recv_exit_status</span><span class="p">()</span>
+ <span class="k">if</span> <span class="n">exit_status</span> <span class="ow">is</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="c1"># returning output if do_xcom_push is set</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">do_xcom_push</span><span class="p">:</span>
+ <span class="n">enable_pickling</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span>
+ <span class="s1">'core'</span><span class="p">,</span> <span class="s1">'enable_xcom_pickling'</span>
+ <span class="p">)</span>
+ <span class="k">if</span> <span class="n">enable_pickling</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">agg_stdout</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">b64encode</span><span class="p">(</span><span class="n">agg_stdout</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span>
+
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">error_msg</span> <span class="o">=</span> <span class="n">agg_stderr</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"error running cmd: </span><span class="si">{0}</span><span class="s2">, error: </span><span class="si">{1}</span><span class="s2">"</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">command</span><span class="p">,</span> <span class="n">error_msg</span><span class="p">))</span>
+
+ <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"SSH operator error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)))</span>
+
+ <span class="k">return</span> <span class="kc">True</span>
+
+ <span class="k">def</span> <span class="nf">tunnel</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="n">ssh_client</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">ssh_client</span><span class="o">.</span><span class="n">get_transport</span><span class="p">()</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/vertica_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/vertica_operator.html b/_modules/airflow/contrib/operators/vertica_operator.html
new file mode 100644
index 0000000..738cb18
--- /dev/null
+++ b/_modules/airflow/contrib/operators/vertica_operator.html
@@ -0,0 +1,284 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.vertica_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.vertica_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.vertica_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.vertica_hook</span> <span class="k">import</span> <span class="n">VerticaHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="VerticaOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.vertica_operator.VerticaOperator">[docs]</a><span class="k">class</span> <span class="nc">VerticaOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Executes sql code in a specific Vertica database</span>
+
+<span class="sd"> :param vertica_conn_id: reference to a specific Vertica database</span>
+<span class="sd"> :type vertica_conn_id: string</span>
+<span class="sd"> :param sql: the sql code to be executed</span>
+<span class="sd"> :type sql: Can receive a str representing a sql statement,</span>
+<span class="sd"> a list of str (sql statements), or reference to a template file.</span>
+<span class="sd"> Template reference are recognized by str ending in '.sql'</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'sql'</span><span class="p">,)</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#b4e0ff'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">vertica_conn_id</span><span class="o">=</span><span class="s1">'vertica_default'</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">VerticaOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">vertica_conn_id</span> <span class="o">=</span> <span class="n">vertica_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">VerticaHook</span><span class="p">(</span><span class="n">vertica_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">vertica_conn_id</span><span class="p">)</span>
+ <span class="n">hook</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[29/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/jenkins_job_trigger_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/jenkins_job_trigger_operator.html b/_modules/airflow/contrib/operators/jenkins_job_trigger_operator.html
new file mode 100644
index 0000000..1bb3657
--- /dev/null
+++ b/_modules/airflow/contrib/operators/jenkins_job_trigger_operator.html
@@ -0,0 +1,484 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.jenkins_job_trigger_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.jenkins_job_trigger_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.jenkins_job_trigger_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">time</span>
+<span class="kn">import</span> <span class="nn">socket</span>
+<span class="kn">import</span> <span class="nn">json</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.jenkins_hook</span> <span class="k">import</span> <span class="n">JenkinsHook</span>
+<span class="kn">import</span> <span class="nn">jenkins</span>
+<span class="kn">from</span> <span class="nn">jenkins</span> <span class="k">import</span> <span class="n">JenkinsException</span>
+<span class="kn">from</span> <span class="nn">six.moves.urllib.request</span> <span class="k">import</span> <span class="n">Request</span><span class="p">,</span> <span class="n">urlopen</span>
+<span class="kn">from</span> <span class="nn">six.moves.urllib.error</span> <span class="k">import</span> <span class="n">HTTPError</span><span class="p">,</span> <span class="n">URLError</span>
+
+<span class="k">try</span><span class="p">:</span>
+ <span class="n">basestring</span>
+<span class="k">except</span> <span class="ne">NameError</span><span class="p">:</span>
+ <span class="n">basestring</span> <span class="o">=</span> <span class="nb">str</span> <span class="c1"># For python3 compatibility</span>
+
+
+<span class="c1"># TODO Use jenkins_urlopen instead when it will be available</span>
+<span class="c1"># in the stable python-jenkins version (> 0.4.15)</span>
+<span class="k">def</span> <span class="nf">jenkins_request_with_headers</span><span class="p">(</span><span class="n">jenkins_server</span><span class="p">,</span> <span class="n">req</span><span class="p">,</span> <span class="n">add_crumb</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> We need to get the headers in addition to the body answer</span>
+<span class="sd"> to get the location from them</span>
+<span class="sd"> This function is just a copy of the one present in python-jenkins library</span>
+<span class="sd"> with just the return call changed</span>
+<span class="sd"> :param jenkins_server: The server to query</span>
+<span class="sd"> :param req: The request to execute</span>
+<span class="sd"> :param add_crumb: Boolean to indicate if it should add crumb to the request</span>
+<span class="sd"> :return:</span>
+<span class="sd"> """</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">jenkins_server</span><span class="o">.</span><span class="n">auth</span><span class="p">:</span>
+ <span class="n">req</span><span class="o">.</span><span class="n">add_header</span><span class="p">(</span><span class="s1">'Authorization'</span><span class="p">,</span> <span class="n">jenkins_server</span><span class="o">.</span><span class="n">auth</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">add_crumb</span><span class="p">:</span>
+ <span class="n">jenkins_server</span><span class="o">.</span><span class="n">maybe_add_crumb</span><span class="p">(</span><span class="n">req</span><span class="p">)</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="n">urlopen</span><span class="p">(</span><span class="n">req</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">jenkins_server</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span>
+ <span class="n">response_body</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
+ <span class="n">response_headers</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">info</span><span class="p">()</span>
+ <span class="k">if</span> <span class="n">response_body</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">jenkins</span><span class="o">.</span><span class="n">EmptyResponseException</span><span class="p">(</span>
+ <span class="s2">"Error communicating with server[</span><span class="si">%s</span><span class="s2">]: "</span>
+ <span class="s2">"empty response"</span> <span class="o">%</span> <span class="n">jenkins_server</span><span class="o">.</span><span class="n">server</span><span class="p">)</span>
+ <span class="k">return</span> <span class="p">{</span><span class="s1">'body'</span><span class="p">:</span> <span class="n">response_body</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">),</span> <span class="s1">'headers'</span><span class="p">:</span> <span class="n">response_headers</span><span class="p">}</span>
+ <span class="k">except</span> <span class="n">HTTPError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="c1"># Jenkins's funky authentication means its nigh impossible to</span>
+ <span class="c1"># distinguish errors.</span>
+ <span class="k">if</span> <span class="n">e</span><span class="o">.</span><span class="n">code</span> <span class="ow">in</span> <span class="p">[</span><span class="mi">401</span><span class="p">,</span> <span class="mi">403</span><span class="p">,</span> <span class="mi">500</span><span class="p">]:</span>
+ <span class="c1"># six.moves.urllib.error.HTTPError provides a 'reason'</span>
+ <span class="c1"># attribute for all python version except for ver 2.6</span>
+ <span class="c1"># Falling back to HTTPError.msg since it contains the</span>
+ <span class="c1"># same info as reason</span>
+ <span class="k">raise</span> <span class="n">JenkinsException</span><span class="p">(</span>
+ <span class="s1">'Error in request. '</span> <span class="o">+</span>
+ <span class="s1">'Possibly authentication failed [</span><span class="si">%s</span><span class="s1">]: </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="p">(</span>
+ <span class="n">e</span><span class="o">.</span><span class="n">code</span><span class="p">,</span> <span class="n">e</span><span class="o">.</span><span class="n">msg</span><span class="p">)</span>
+ <span class="p">)</span>
+ <span class="k">elif</span> <span class="n">e</span><span class="o">.</span><span class="n">code</span> <span class="o">==</span> <span class="mi">404</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">jenkins</span><span class="o">.</span><span class="n">NotFoundException</span><span class="p">(</span><span class="s1">'Requested item could not be found'</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span>
+ <span class="k">except</span> <span class="n">socket</span><span class="o">.</span><span class="n">timeout</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">jenkins</span><span class="o">.</span><span class="n">TimeoutException</span><span class="p">(</span><span class="s1">'Error in request: </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">e</span><span class="p">)</span>
+ <span class="k">except</span> <span class="n">URLError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="c1"># python 2.6 compatibility to ensure same exception raised</span>
+ <span class="c1"># since URLError wraps a socket timeout on python 2.6.</span>
+ <span class="k">if</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">reason</span><span class="p">)</span> <span class="o">==</span> <span class="s2">"timed out"</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">jenkins</span><span class="o">.</span><span class="n">TimeoutException</span><span class="p">(</span><span class="s1">'Error in request: </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">e</span><span class="o">.</span><span class="n">reason</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="n">JenkinsException</span><span class="p">(</span><span class="s1">'Error in request: </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">e</span><span class="o">.</span><span class="n">reason</span><span class="p">)</span>
+
+
+<div class="viewcode-block" id="JenkinsJobTriggerOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.jenkins_job_trigger_operator.JenkinsJobTriggerOperator">[docs]</a><span class="k">class</span> <span class="nc">JenkinsJobTriggerOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Trigger a Jenkins Job and monitor it's execution.</span>
+<span class="sd"> This operator depend on python-jenkins library,</span>
+<span class="sd"> version >= 0.4.15 to communicate with jenkins server.</span>
+<span class="sd"> You'll also need to configure a Jenkins connection in the connections screen.</span>
+<span class="sd"> :param jenkins_connection_id: The jenkins connection to use for this job</span>
+<span class="sd"> :type jenkins_connection_id: string</span>
+<span class="sd"> :param job_name: The name of the job to trigger</span>
+<span class="sd"> :type job_name: string</span>
+<span class="sd"> :param parameters: The parameters block to provide to jenkins</span>
+<span class="sd"> :type parameters: string</span>
+<span class="sd"> :param sleep_time: How long will the operator sleep between each status</span>
+<span class="sd"> request for the job (min 1, default 10)</span>
+<span class="sd"> :type sleep_time: int</span>
+<span class="sd"> :param max_try_before_job_appears: The maximum number of requests to make</span>
+<span class="sd"> while waiting for the job to appears on jenkins server (default 10)</span>
+<span class="sd"> :type max_try_before_job_appears: int</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'parameters'</span><span class="p">,)</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.json'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f9ec86'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">jenkins_connection_id</span><span class="p">,</span>
+ <span class="n">job_name</span><span class="p">,</span>
+ <span class="n">parameters</span><span class="o">=</span><span class="s2">""</span><span class="p">,</span>
+ <span class="n">sleep_time</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span>
+ <span class="n">max_try_before_job_appears</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">JenkinsJobTriggerOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job_name</span> <span class="o">=</span> <span class="n">job_name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span> <span class="o">=</span> <span class="n">parameters</span>
+ <span class="k">if</span> <span class="n">sleep_time</span> <span class="o"><</span> <span class="mi">1</span><span class="p">:</span>
+ <span class="n">sleep_time</span> <span class="o">=</span> <span class="mi">1</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sleep_time</span> <span class="o">=</span> <span class="n">sleep_time</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">jenkins_connection_id</span> <span class="o">=</span> <span class="n">jenkins_connection_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">max_try_before_job_appears</span> <span class="o">=</span> <span class="n">max_try_before_job_appears</span>
+
+<div class="viewcode-block" id="JenkinsJobTriggerOperator.build_job"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.jenkins_job_trigger_operator.JenkinsJobTriggerOperator.build_job">[docs]</a> <span class="k">def</span> <span class="nf">build_job</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">jenkins_server</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> This function makes an API call to Jenkins to trigger a build for 'job_name'</span>
+<span class="sd"> It returned a dict with 2 keys : body and headers.</span>
+<span class="sd"> headers contains also a dict-like object which can be queried to get</span>
+<span class="sd"> the location to poll in the queue.</span>
+<span class="sd"> :param jenkins_server: The jenkins server where the job should be triggered</span>
+<span class="sd"> :return: Dict containing the response body (key body)</span>
+<span class="sd"> and the headers coming along (headers)</span>
+<span class="sd"> """</span>
+ <span class="c1"># Warning if the parameter is too long, the URL can be longer than</span>
+ <span class="c1"># the maximum allowed size</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span> <span class="ow">and</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">parameters</span><span class="p">,</span> <span class="n">basestring</span><span class="p">):</span>
+ <span class="kn">import</span> <span class="nn">ast</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span> <span class="o">=</span> <span class="n">ast</span><span class="o">.</span><span class="n">literal_eval</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">parameters</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span><span class="p">:</span>
+ <span class="c1"># We need a None to call the non parametrized jenkins api end point</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span> <span class="o">=</span> <span class="kc">None</span>
+
+ <span class="n">request</span> <span class="o">=</span> <span class="n">Request</span><span class="p">(</span><span class="n">jenkins_server</span><span class="o">.</span><span class="n">build_job_url</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span> <span class="sa">b</span><span class="s1">''</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">jenkins_request_with_headers</span><span class="p">(</span><span class="n">jenkins_server</span><span class="p">,</span> <span class="n">request</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="JenkinsJobTriggerOperator.poll_job_in_queue"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.jenkins_job_trigger_operator.JenkinsJobTriggerOperator.poll_job_in_queue">[docs]</a> <span class="k">def</span> <span class="nf">poll_job_in_queue</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">location</span><span class="p">,</span> <span class="n">jenkins_server</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> This method poll the jenkins queue until the job is executed.</span>
+<span class="sd"> When we trigger a job through an API call,</span>
+<span class="sd"> the job is first put in the queue without having a build number assigned.</span>
+<span class="sd"> Thus we have to wait the job exit the queue to know its build number.</span>
+<span class="sd"> To do so, we have to add /api/json (or /api/xml) to the location</span>
+<span class="sd"> returned by the build_job call and poll this file.</span>
+<span class="sd"> When a 'executable' block appears in the json, it means the job execution started</span>
+<span class="sd"> and the field 'number' then contains the build number.</span>
+<span class="sd"> :param location: Location to poll, returned in the header of the build_job call</span>
+<span class="sd"> :param jenkins_server: The jenkins server to poll</span>
+<span class="sd"> :return: The build_number corresponding to the triggered job</span>
+<span class="sd"> """</span>
+ <span class="n">try_count</span> <span class="o">=</span> <span class="mi">0</span>
+ <span class="n">location</span> <span class="o">=</span> <span class="n">location</span> <span class="o">+</span> <span class="s1">'/api/json'</span>
+ <span class="c1"># TODO Use get_queue_info instead</span>
+ <span class="c1"># once it will be available in python-jenkins (v > 0.4.15)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Polling jenkins queue at the url </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">location</span><span class="p">)</span>
+ <span class="k">while</span> <span class="n">try_count</span> <span class="o"><</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_try_before_job_appears</span><span class="p">:</span>
+ <span class="n">location_answer</span> <span class="o">=</span> <span class="n">jenkins_request_with_headers</span><span class="p">(</span><span class="n">jenkins_server</span><span class="p">,</span>
+ <span class="n">Request</span><span class="p">(</span><span class="n">location</span><span class="p">))</span>
+ <span class="k">if</span> <span class="n">location_answer</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">json_response</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">location_answer</span><span class="p">[</span><span class="s1">'body'</span><span class="p">])</span>
+ <span class="k">if</span> <span class="s1">'executable'</span> <span class="ow">in</span> <span class="n">json_response</span><span class="p">:</span>
+ <span class="n">build_number</span> <span class="o">=</span> <span class="n">json_response</span><span class="p">[</span><span class="s1">'executable'</span><span class="p">][</span><span class="s1">'number'</span><span class="p">]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Job executed on Jenkins side with the build number </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">build_number</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">build_number</span>
+ <span class="n">try_count</span> <span class="o">+=</span> <span class="mi">1</span>
+ <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sleep_time</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"The job hasn't been executed"</span>
+ <span class="s2">" after polling the queue </span><span class="si">%d</span><span class="s2"> times"</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">max_try_before_job_appears</span><span class="p">)</span></div>
+
+ <span class="k">def</span> <span class="nf">get_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">JenkinsHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">jenkins_connection_id</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">jenkins_connection_id</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
+ <span class="s1">'Please specify the jenkins connection id to use.'</span>
+ <span class="s1">'You must create a Jenkins connection before'</span>
+ <span class="s1">' being able to use this operator'</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'The jenkins_connection_id parameter is missing,'</span>
+ <span class="s1">'impossible to trigger the job'</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Please specify the job name to use in the job_name parameter"</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'The job_name parameter is missing,'</span>
+ <span class="s1">'impossible to trigger the job'</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'Triggering the job </span><span class="si">%s</span><span class="s1"> on the jenkins : </span><span class="si">%s</span><span class="s1"> with the parameters : </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">jenkins_connection_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span><span class="p">)</span>
+ <span class="n">jenkins_server</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span><span class="o">.</span><span class="n">get_jenkins_server</span><span class="p">()</span>
+ <span class="n">jenkins_response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">build_job</span><span class="p">(</span><span class="n">jenkins_server</span><span class="p">)</span>
+ <span class="n">build_number</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">poll_job_in_queue</span><span class="p">(</span>
+ <span class="n">jenkins_response</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'Location'</span><span class="p">],</span> <span class="n">jenkins_server</span><span class="p">)</span>
+
+ <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sleep_time</span><span class="p">)</span>
+ <span class="n">keep_polling_job</span> <span class="o">=</span> <span class="kc">True</span>
+ <span class="n">build_info</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="k">while</span> <span class="n">keep_polling_job</span><span class="p">:</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">build_info</span> <span class="o">=</span> <span class="n">jenkins_server</span><span class="o">.</span><span class="n">get_build_info</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">,</span>
+ <span class="n">number</span><span class="o">=</span><span class="n">build_number</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">build_info</span><span class="p">[</span><span class="s1">'result'</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">keep_polling_job</span> <span class="o">=</span> <span class="kc">False</span>
+ <span class="c1"># Check if job had errors.</span>
+ <span class="k">if</span> <span class="n">build_info</span><span class="p">[</span><span class="s1">'result'</span><span class="p">]</span> <span class="o">!=</span> <span class="s1">'SUCCESS'</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'Jenkins job failed, final state : </span><span class="si">%s</span><span class="s1">.'</span>
+ <span class="s1">'Find more information on job url : </span><span class="si">%s</span><span class="s1">'</span>
+ <span class="o">%</span> <span class="p">(</span><span class="n">build_info</span><span class="p">[</span><span class="s1">'result'</span><span class="p">],</span> <span class="n">build_info</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]))</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Waiting for job to complete : </span><span class="si">%s</span><span class="s1"> , build </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">,</span> <span class="n">build_number</span><span class="p">)</span>
+ <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sleep_time</span><span class="p">)</span>
+ <span class="k">except</span> <span class="n">jenkins</span><span class="o">.</span><span class="n">NotFoundException</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'Jenkins job status check failed. Final error was: </span><span class="si">%s</span><span class="s1">'</span>
+ <span class="o">%</span> <span class="n">err</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span><span class="p">)</span>
+ <span class="k">except</span> <span class="n">jenkins</span><span class="o">.</span><span class="n">JenkinsException</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'Jenkins call failed with error : </span><span class="si">%s</span><span class="s1">, if you have parameters '</span>
+ <span class="s1">'double check them, jenkins sends back '</span>
+ <span class="s1">'this exception for unknown parameters'</span>
+ <span class="s1">'You can also check logs for more details on this exception '</span>
+ <span class="s1">'(jenkins_url/log/rss)'</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">err</span><span class="p">))</span>
+ <span class="k">if</span> <span class="n">build_info</span><span class="p">:</span>
+ <span class="c1"># If we can we return the url of the job</span>
+ <span class="c1"># for later use (like retrieving an artifact)</span>
+ <span class="k">return</span> <span class="n">build_info</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/jira_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/jira_operator.html b/_modules/airflow/contrib/operators/jira_operator.html
new file mode 100644
index 0000000..155c46d
--- /dev/null
+++ b/_modules/airflow/contrib/operators/jira_operator.html
@@ -0,0 +1,329 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.jira_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.jira_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.jira_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.jira_hook</span> <span class="k">import</span> <span class="n">JIRAError</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.jira_hook</span> <span class="k">import</span> <span class="n">JiraHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="JiraOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.jira_operator.JiraOperator">[docs]</a><span class="k">class</span> <span class="nc">JiraOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> JiraOperator to interact and perform action on Jira issue tracking system.</span>
+<span class="sd"> This operator is designed to use Jira Python SDK: http://jira.readthedocs.io</span>
+
+<span class="sd"> :param jira_conn_id: reference to a pre-defined Jira Connection</span>
+<span class="sd"> :type jira_conn_id: str</span>
+<span class="sd"> :param jira_method: method name from Jira Python SDK to be called</span>
+<span class="sd"> :type jira_method: str</span>
+<span class="sd"> :param jira_method_args: required method parameters for the jira_method</span>
+<span class="sd"> :type jira_method_args: dict</span>
+<span class="sd"> :param result_processor: function to further process the response from Jira</span>
+<span class="sd"> :type result_processor: function</span>
+<span class="sd"> :param get_jira_resource_method: function or operator to get jira resource</span>
+<span class="sd"> on which the provided jira_method will be executed</span>
+<span class="sd"> :type get_jira_resource_method: function</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s2">"jira_method_args"</span><span class="p">,)</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">jira_conn_id</span><span class="o">=</span><span class="s1">'jira_default'</span><span class="p">,</span>
+ <span class="n">jira_method</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">jira_method_args</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">result_processor</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">get_jira_resource_method</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">JiraOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">jira_conn_id</span> <span class="o">=</span> <span class="n">jira_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">method_name</span> <span class="o">=</span> <span class="n">jira_method</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">jira_method_args</span> <span class="o">=</span> <span class="n">jira_method_args</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">result_processor</span> <span class="o">=</span> <span class="n">result_processor</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">get_jira_resource_method</span> <span class="o">=</span> <span class="n">get_jira_resource_method</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_jira_resource_method</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="c1"># if get_jira_resource_method is provided, jira_method will be executed on</span>
+ <span class="c1"># resource returned by executing the get_jira_resource_method.</span>
+ <span class="c1"># This makes all the provided methods of JIRA sdk accessible and usable</span>
+ <span class="c1"># directly at the JiraOperator without additional wrappers.</span>
+ <span class="c1"># ref: http://jira.readthedocs.io/en/latest/api.html</span>
+ <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_jira_resource_method</span><span class="p">,</span> <span class="n">JiraOperator</span><span class="p">):</span>
+ <span class="n">resource</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_jira_resource_method</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="o">**</span><span class="n">context</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">resource</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_jira_resource_method</span><span class="p">(</span><span class="o">**</span><span class="n">context</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="c1"># Default method execution is on the top level jira client resource</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">JiraHook</span><span class="p">(</span><span class="n">jira_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">jira_conn_id</span><span class="p">)</span>
+ <span class="n">resource</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">client</span>
+
+ <span class="c1"># Current Jira-Python SDK (1.0.7) has issue with pickling the jira response.</span>
+ <span class="c1"># ex: self.xcom_push(context, key='operator_response', value=jira_response)</span>
+ <span class="c1"># This could potentially throw error if jira_result is not picklable</span>
+ <span class="n">jira_result</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">resource</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">method_name</span><span class="p">)(</span><span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">jira_method_args</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_processor</span><span class="p">:</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_processor</span><span class="p">(</span><span class="n">context</span><span class="p">,</span> <span class="n">jira_result</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">jira_result</span>
+
+ <span class="k">except</span> <span class="n">JIRAError</span> <span class="k">as</span> <span class="n">jira_error</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Failed to execute jiraOperator, error: </span><span class="si">%s</span><span class="s2">"</span>
+ <span class="o">%</span> <span class="nb">str</span><span class="p">(</span><span class="n">jira_error</span><span class="p">))</span>
+ <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Jira operator error: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/kubernetes_pod_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/kubernetes_pod_operator.html b/_modules/airflow/contrib/operators/kubernetes_pod_operator.html
new file mode 100644
index 0000000..07b5378
--- /dev/null
+++ b/_modules/airflow/contrib/operators/kubernetes_pod_operator.html
@@ -0,0 +1,362 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.kubernetes_pod_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.kubernetes_pod_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.kubernetes_pod_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.kubernetes</span> <span class="k">import</span> <span class="n">kube_client</span><span class="p">,</span> <span class="n">pod_generator</span><span class="p">,</span> <span class="n">pod_launcher</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.kubernetes.pod</span> <span class="k">import</span> <span class="n">Resources</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span>
+
+<span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'templates_dict'</span><span class="p">,)</span>
+<span class="n">template_ext</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">()</span>
+<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#ffefeb'</span>
+
+<div class="viewcode-block" id="KubernetesPodOperator"><a class="viewcode-back" href="../../../../kubernetes.html#airflow.contrib.operators.kubernetes_pod_operator.KubernetesPodOperator">[docs]</a><span class="k">class</span> <span class="nc">KubernetesPodOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Execute a task in a Kubernetes Pod</span>
+
+<span class="sd"> :param image: Docker image you wish to launch. Defaults to dockerhub.io,</span>
+<span class="sd"> but fully qualified URLS will point to custom repositories</span>
+<span class="sd"> :type image: str</span>
+<span class="sd"> :param: namespace: the namespace to run within kubernetes</span>
+<span class="sd"> :type: namespace: str</span>
+<span class="sd"> :param cmds: entrypoint of the container.</span>
+<span class="sd"> The docker images's entrypoint is used if this is not provide.</span>
+<span class="sd"> :type cmds: list</span>
+<span class="sd"> :param arguments: arguments of to the entrypoint.</span>
+<span class="sd"> The docker image's CMD is used if this is not provided.</span>
+<span class="sd"> :type arguments: list</span>
+<span class="sd"> :param labels: labels to apply to the Pod</span>
+<span class="sd"> :type labels: dict</span>
+<span class="sd"> :param startup_timeout_seconds: timeout in seconds to startup the pod</span>
+<span class="sd"> :type startup_timeout_seconds: int</span>
+<span class="sd"> :param name: name of the task you want to run,</span>
+<span class="sd"> will be used to generate a pod id</span>
+<span class="sd"> :type name: str</span>
+<span class="sd"> :param env_vars: Environment variables initialized in the container</span>
+<span class="sd"> :type env_vars: dict</span>
+<span class="sd"> :param secrets: Kubernetes secrets to inject in the container,</span>
+<span class="sd"> They can be exposed as environment vars or files in a volume.</span>
+<span class="sd"> :type secrets: list</span>
+<span class="sd"> :param in_cluster: run kubernetes client with in_cluster configuration</span>
+<span class="sd"> :type in_cluster: bool</span>
+<span class="sd"> :param get_logs: get the stdout of the container as logs of the tasks</span>
+<span class="sd"> :type get_logs: bool</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'cmds'</span><span class="p">,</span> <span class="s1">'arguments'</span><span class="p">,</span> <span class="s1">'env_vars'</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">client</span> <span class="o">=</span> <span class="n">kube_client</span><span class="o">.</span><span class="n">get_kube_client</span><span class="p">(</span><span class="n">in_cluster</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">in_cluster</span><span class="p">)</span>
+ <span class="n">gen</span> <span class="o">=</span> <span class="n">pod_generator</span><span class="o">.</span><span class="n">PodGenerator</span><span class="p">()</span>
+
+ <span class="n">pod</span> <span class="o">=</span> <span class="n">gen</span><span class="o">.</span><span class="n">make_pod</span><span class="p">(</span>
+ <span class="n">namespace</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">namespace</span><span class="p">,</span>
+ <span class="n">image</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">image</span><span class="p">,</span>
+ <span class="n">pod_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">,</span>
+ <span class="n">cmds</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cmds</span><span class="p">,</span>
+ <span class="n">arguments</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">arguments</span><span class="p">,</span>
+ <span class="n">labels</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">labels</span>
+ <span class="p">)</span>
+
+ <span class="n">pod</span><span class="o">.</span><span class="n">secrets</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">secrets</span>
+ <span class="n">pod</span><span class="o">.</span><span class="n">envs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">env_vars</span>
+ <span class="n">pod</span><span class="o">.</span><span class="n">image_pull_policy</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">image_pull_policy</span>
+ <span class="n">pod</span><span class="o">.</span><span class="n">annotations</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">annotations</span>
+ <span class="n">pod</span><span class="o">.</span><span class="n">resources</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">resources</span>
+
+ <span class="n">launcher</span> <span class="o">=</span> <span class="n">pod_launcher</span><span class="o">.</span><span class="n">PodLauncher</span><span class="p">(</span><span class="n">client</span><span class="p">)</span>
+ <span class="n">final_state</span> <span class="o">=</span> <span class="n">launcher</span><span class="o">.</span><span class="n">run_pod</span><span class="p">(</span>
+ <span class="n">pod</span><span class="p">,</span>
+ <span class="n">startup_timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">startup_timeout_seconds</span><span class="p">,</span>
+ <span class="n">get_logs</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">get_logs</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">final_state</span> <span class="o">!=</span> <span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'Pod returned a failure: </span><span class="si">{state}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">state</span><span class="o">=</span><span class="n">final_state</span><span class="p">)</span>
+ <span class="p">)</span>
+ <span class="k">except</span> <span class="n">AirflowException</span> <span class="k">as</span> <span class="n">ex</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Pod Launching failed: </span><span class="si">{error}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">=</span><span class="n">ex</span><span class="p">))</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">namespace</span><span class="p">,</span>
+ <span class="n">image</span><span class="p">,</span>
+ <span class="n">name</span><span class="p">,</span>
+ <span class="n">cmds</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">arguments</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">env_vars</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">secrets</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">in_cluster</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">startup_timeout_seconds</span><span class="o">=</span><span class="mi">120</span><span class="p">,</span>
+ <span class="n">get_logs</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+ <span class="n">image_pull_policy</span><span class="o">=</span><span class="s1">'IfNotPresent'</span><span class="p">,</span>
+ <span class="n">annotations</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">resources</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">KubernetesPodOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">image</span> <span class="o">=</span> <span class="n">image</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">namespace</span> <span class="o">=</span> <span class="n">namespace</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cmds</span> <span class="o">=</span> <span class="n">cmds</span> <span class="ow">or</span> <span class="p">[]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">arguments</span> <span class="o">=</span> <span class="n">arguments</span> <span class="ow">or</span> <span class="p">[]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="o">=</span> <span class="n">labels</span> <span class="ow">or</span> <span class="p">{}</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">startup_timeout_seconds</span> <span class="o">=</span> <span class="n">startup_timeout_seconds</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">env_vars</span> <span class="o">=</span> <span class="n">env_vars</span> <span class="ow">or</span> <span class="p">{}</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">secrets</span> <span class="o">=</span> <span class="n">secrets</span> <span class="ow">or</span> <span class="p">[]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">in_cluster</span> <span class="o">=</span> <span class="n">in_cluster</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">get_logs</span> <span class="o">=</span> <span class="n">get_logs</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">image_pull_policy</span> <span class="o">=</span> <span class="n">image_pull_policy</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">annotations</span> <span class="o">=</span> <span class="n">annotations</span> <span class="ow">or</span> <span class="p">{}</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">resources</span> <span class="o">=</span> <span class="n">resources</span> <span class="ow">or</span> <span class="n">Resources</span><span class="p">()</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[05/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/slack_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/slack_operator.html b/_modules/airflow/operators/slack_operator.html
new file mode 100644
index 0000000..40152da
--- /dev/null
+++ b/_modules/airflow/operators/slack_operator.html
@@ -0,0 +1,366 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.operators.slack_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.operators.slack_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.operators.slack_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">json</span>
+
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.slack_hook</span> <span class="k">import</span> <span class="n">SlackHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+
+
+<div class="viewcode-block" id="SlackAPIOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.slack_operator.SlackAPIOperator">[docs]</a><span class="k">class</span> <span class="nc">SlackAPIOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Base Slack Operator</span>
+<span class="sd"> The SlackAPIPostOperator is derived from this operator.</span>
+<span class="sd"> In the future additional Slack API Operators will be derived from this class as well</span>
+
+<span class="sd"> :param slack_conn_id: Slack connection ID which its password is Slack API token</span>
+<span class="sd"> :type slack_conn_id: string</span>
+<span class="sd"> :param token: Slack API token (https://api.slack.com/web)</span>
+<span class="sd"> :type token: string</span>
+<span class="sd"> :param method: The Slack API Method to Call (https://api.slack.com/methods)</span>
+<span class="sd"> :type method: string</span>
+<span class="sd"> :param api_params: API Method call parameters (https://api.slack.com/methods)</span>
+<span class="sd"> :type api_params: dict</span>
+<span class="sd"> """</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">slack_conn_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">token</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">method</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">api_params</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">SlackAPIOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">token</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">slack_conn_id</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'No valid Slack token nor slack_conn_id supplied.'</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">token</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">slack_conn_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Cannot determine Slack credential when both token and slack_conn_id are supplied.'</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">token</span> <span class="o">=</span> <span class="n">token</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">slack_conn_id</span> <span class="o">=</span> <span class="n">slack_conn_id</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">=</span> <span class="n">method</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">api_params</span> <span class="o">=</span> <span class="n">api_params</span>
+
+<div class="viewcode-block" id="SlackAPIOperator.construct_api_call_params"><a class="viewcode-back" href="../../../code.html#airflow.operators.slack_operator.SlackAPIOperator.construct_api_call_params">[docs]</a> <span class="k">def</span> <span class="nf">construct_api_call_params</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Used by the execute function. Allows templating on the source fields of the api_call_params dict before construction</span>
+
+<span class="sd"> Override in child classes.</span>
+<span class="sd"> Each SlackAPIOperator child class is responsible for having a construct_api_call_params function</span>
+<span class="sd"> which sets self.api_call_params with a dict of API call parameters (https://api.slack.com/methods)</span>
+<span class="sd"> """</span>
+
+ <span class="k">pass</span></div>
+
+<div class="viewcode-block" id="SlackAPIOperator.execute"><a class="viewcode-back" href="../../../code.html#airflow.operators.slack_operator.SlackAPIOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> SlackAPIOperator calls will not fail even if the call is not unsuccessful.</span>
+<span class="sd"> It should not prevent a DAG from completing in success</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">api_params</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">construct_api_call_params</span><span class="p">()</span>
+ <span class="n">slack</span> <span class="o">=</span> <span class="n">SlackHook</span><span class="p">(</span><span class="n">token</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">token</span><span class="p">,</span> <span class="n">slack_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">slack_conn_id</span><span class="p">)</span>
+ <span class="n">slack</span><span class="o">.</span><span class="n">call</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">method</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">api_params</span><span class="p">)</span></div></div>
+
+
+<div class="viewcode-block" id="SlackAPIPostOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.slack_operator.SlackAPIPostOperator">[docs]</a><span class="k">class</span> <span class="nc">SlackAPIPostOperator</span><span class="p">(</span><span class="n">SlackAPIOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Posts messages to a slack channel</span>
+
+<span class="sd"> :param channel: channel in which to post message on slack name (#general) or ID (C12318391)</span>
+<span class="sd"> :type channel: string</span>
+<span class="sd"> :param username: Username that airflow will be posting to Slack as</span>
+<span class="sd"> :type username: string</span>
+<span class="sd"> :param text: message to send to slack</span>
+<span class="sd"> :type text: string</span>
+<span class="sd"> :param icon_url: url to icon used for this message</span>
+<span class="sd"> :type icon_url: string</span>
+<span class="sd"> :param attachments: extra formatting details - see https://api.slack.com/docs/attachments</span>
+<span class="sd"> :type attachments: array of hashes</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'username'</span><span class="p">,</span> <span class="s1">'text'</span><span class="p">,</span> <span class="s1">'attachments'</span><span class="p">,</span> <span class="s1">'channel'</span><span class="p">)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#FFBA40'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">channel</span><span class="o">=</span><span class="s1">'#general'</span><span class="p">,</span>
+ <span class="n">username</span><span class="o">=</span><span class="s1">'Airflow'</span><span class="p">,</span>
+ <span class="n">text</span><span class="o">=</span><span class="s1">'No message has been set.</span><span class="se">\n</span><span class="s1">'</span>
+ <span class="s1">'Here is a cat video instead</span><span class="se">\n</span><span class="s1">'</span>
+ <span class="s1">'https://www.youtube.com/watch?v=J---aiyznGQ'</span><span class="p">,</span>
+ <span class="n">icon_url</span><span class="o">=</span><span class="s1">'https://raw.githubusercontent.com/airbnb/airflow/master/airflow/www/static/pin_100.png'</span><span class="p">,</span>
+ <span class="n">attachments</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">=</span> <span class="s1">'chat.postMessage'</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">channel</span> <span class="o">=</span> <span class="n">channel</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="o">=</span> <span class="n">username</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">text</span> <span class="o">=</span> <span class="n">text</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">icon_url</span> <span class="o">=</span> <span class="n">icon_url</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">attachments</span> <span class="o">=</span> <span class="n">attachments</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">SlackAPIPostOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">method</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">method</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">construct_api_call_params</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">api_params</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'channel'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">channel</span><span class="p">,</span>
+ <span class="s1">'username'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">,</span>
+ <span class="s1">'text'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">text</span><span class="p">,</span>
+ <span class="s1">'icon_url'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">icon_url</span><span class="p">,</span>
+ <span class="s1">'attachments'</span><span class="p">:</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">attachments</span><span class="p">),</span>
+ <span class="p">}</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/sqlite_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/sqlite_operator.html b/_modules/airflow/operators/sqlite_operator.html
new file mode 100644
index 0000000..8bdcd6e
--- /dev/null
+++ b/_modules/airflow/operators/sqlite_operator.html
@@ -0,0 +1,286 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.operators.sqlite_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.operators.sqlite_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.operators.sqlite_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.sqlite_hook</span> <span class="k">import</span> <span class="n">SqliteHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="SqliteOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.sqlite_operator.SqliteOperator">[docs]</a><span class="k">class</span> <span class="nc">SqliteOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Executes sql code in a specific Sqlite database</span>
+
+<span class="sd"> :param sqlite_conn_id: reference to a specific sqlite database</span>
+<span class="sd"> :type sqlite_conn_id: string</span>
+<span class="sd"> :param sql: the sql code to be executed</span>
+<span class="sd"> :type sql: string or string pointing to a template file. File must have</span>
+<span class="sd"> a '.sql' extensions.</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'sql'</span><span class="p">,)</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#cdaaed'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">sqlite_conn_id</span><span class="o">=</span><span class="s1">'sqlite_default'</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">SqliteOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sqlite_conn_id</span> <span class="o">=</span> <span class="n">sqlite_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span> <span class="o">=</span> <span class="n">parameters</span> <span class="ow">or</span> <span class="p">[]</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">SqliteHook</span><span class="p">(</span><span class="n">sqlite_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">sqlite_conn_id</span><span class="p">)</span>
+ <span class="n">hook</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">parameters</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/subdag_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/subdag_operator.html b/_modules/airflow/operators/subdag_operator.html
new file mode 100644
index 0000000..03b49c3
--- /dev/null
+++ b/_modules/airflow/operators/subdag_operator.html
@@ -0,0 +1,338 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.operators.subdag_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.operators.subdag_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.operators.subdag_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.executors.sequential_executor</span> <span class="k">import</span> <span class="n">SequentialExecutor</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span><span class="p">,</span> <span class="n">Pool</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.db</span> <span class="k">import</span> <span class="n">provide_session</span>
+
+
+<div class="viewcode-block" id="SubDagOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.subdag_operator.SubDagOperator">[docs]</a><span class="k">class</span> <span class="nc">SubDagOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">()</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#555'</span>
+ <span class="n">ui_fgcolor</span> <span class="o">=</span> <span class="s1">'#fff'</span>
+
+ <span class="nd">@provide_session</span>
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">subdag</span><span class="p">,</span>
+ <span class="n">executor</span><span class="o">=</span><span class="n">SequentialExecutor</span><span class="p">(),</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> This runs a sub dag. By convention, a sub dag's dag_id</span>
+<span class="sd"> should be prefixed by its parent and a dot. As in `parent.child`.</span>
+
+<span class="sd"> :param subdag: the DAG object to run as a subdag of the current DAG.</span>
+<span class="sd"> :type subdag: airflow.DAG.</span>
+<span class="sd"> :param dag: the parent DAG for the subdag.</span>
+<span class="sd"> :type dag: airflow.DAG.</span>
+<span class="sd"> :param executor: the executor for this subdag. Default to use SequentialExecutor.</span>
+<span class="sd"> Please find AIRFLOW-74 for more details.</span>
+<span class="sd"> :type executor: airflow.executors.</span>
+<span class="sd"> """</span>
+ <span class="kn">import</span> <span class="nn">airflow.models</span>
+ <span class="n">dag</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'dag'</span><span class="p">)</span> <span class="ow">or</span> <span class="n">airflow</span><span class="o">.</span><span class="n">models</span><span class="o">.</span><span class="n">_CONTEXT_MANAGER_DAG</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">dag</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Please pass in the `dag` param or call '</span>
+ <span class="s1">'within a DAG context manager'</span><span class="p">)</span>
+ <span class="n">session</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s1">'session'</span><span class="p">)</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">SubDagOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="c1"># validate subdag name</span>
+ <span class="k">if</span> <span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">+</span> <span class="s1">'.'</span> <span class="o">+</span> <span class="n">kwargs</span><span class="p">[</span><span class="s1">'task_id'</span><span class="p">]</span> <span class="o">!=</span> <span class="n">subdag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s2">"The subdag's dag_id should have the form "</span>
+ <span class="s2">"'{{parent_dag_id}}.{{this_task_id}}'. Expected "</span>
+ <span class="s2">"'</span><span class="si">{d}</span><span class="s2">.</span><span class="si">{t}</span><span class="s2">'; received '</span><span class="si">{rcvd}</span><span class="s2">'."</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">d</span><span class="o">=</span><span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">t</span><span class="o">=</span><span class="n">kwargs</span><span class="p">[</span><span class="s1">'task_id'</span><span class="p">],</span> <span class="n">rcvd</span><span class="o">=</span><span class="n">subdag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">))</span>
+
+ <span class="c1"># validate that subdag operator and subdag tasks don't have a</span>
+ <span class="c1"># pool conflict</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">pool</span><span class="p">:</span>
+ <span class="n">conflicts</span> <span class="o">=</span> <span class="p">[</span><span class="n">t</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">subdag</span><span class="o">.</span><span class="n">tasks</span> <span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">pool</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">pool</span><span class="p">]</span>
+ <span class="k">if</span> <span class="n">conflicts</span><span class="p">:</span>
+ <span class="c1"># only query for pool conflicts if one may exist</span>
+ <span class="n">pool</span> <span class="o">=</span> <span class="p">(</span>
+ <span class="n">session</span>
+ <span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">Pool</span><span class="p">)</span>
+ <span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">Pool</span><span class="o">.</span><span class="n">slots</span> <span class="o">==</span> <span class="mi">1</span><span class="p">)</span>
+ <span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">Pool</span><span class="o">.</span><span class="n">pool</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">pool</span><span class="p">)</span>
+ <span class="o">.</span><span class="n">first</span><span class="p">()</span>
+ <span class="p">)</span>
+ <span class="k">if</span> <span class="n">pool</span> <span class="ow">and</span> <span class="nb">any</span><span class="p">(</span><span class="n">t</span><span class="o">.</span><span class="n">pool</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">pool</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">subdag</span><span class="o">.</span><span class="n">tasks</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'SubDagOperator </span><span class="si">{sd}</span><span class="s1"> and subdag task</span><span class="si">{plural}</span><span class="s1"> </span><span class="si">{t}</span><span class="s1"> both '</span>
+ <span class="s1">'use pool </span><span class="si">{p}</span><span class="s1">, but the pool only has 1 slot. The '</span>
+ <span class="s1">'subdag tasks will never run.'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">sd</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span>
+ <span class="n">plural</span><span class="o">=</span><span class="nb">len</span><span class="p">(</span><span class="n">conflicts</span><span class="p">)</span> <span class="o">></span> <span class="mi">1</span><span class="p">,</span>
+ <span class="n">t</span><span class="o">=</span><span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">t</span><span class="o">.</span><span class="n">task_id</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">conflicts</span><span class="p">),</span>
+ <span class="n">p</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">pool</span>
+ <span class="p">)</span>
+ <span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">subdag</span> <span class="o">=</span> <span class="n">subdag</span>
+ <span class="c1"># Airflow pool is not honored by SubDagOperator.</span>
+ <span class="c1"># Hence resources could be consumed by SubdagOperators</span>
+ <span class="c1"># Use other executor with your own risk.</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">executor</span> <span class="o">=</span> <span class="n">executor</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">ed</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">'execution_date'</span><span class="p">]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">subdag</span><span class="o">.</span><span class="n">run</span><span class="p">(</span>
+ <span class="n">start_date</span><span class="o">=</span><span class="n">ed</span><span class="p">,</span> <span class="n">end_date</span><span class="o">=</span><span class="n">ed</span><span class="p">,</span> <span class="n">donot_pickle</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+ <span class="n">executor</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">executor</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/sensors/base_sensor_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/sensors/base_sensor_operator.html b/_modules/airflow/sensors/base_sensor_operator.html
new file mode 100644
index 0000000..d7b8d0c
--- /dev/null
+++ b/_modules/airflow/sensors/base_sensor_operator.html
@@ -0,0 +1,310 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.sensors.base_sensor_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.sensors.base_sensor_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.sensors.base_sensor_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+
+<span class="kn">from</span> <span class="nn">time</span> <span class="k">import</span> <span class="n">sleep</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span><span class="p">,</span> <span class="n">AirflowSensorTimeout</span><span class="p">,</span> \
+ <span class="n">AirflowSkipException</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils</span> <span class="k">import</span> <span class="n">timezone</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="BaseSensorOperator"><a class="viewcode-back" href="../../../code.html#airflow.sensors.base_sensor_operator.BaseSensorOperator">[docs]</a><span class="k">class</span> <span class="nc">BaseSensorOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Sensor operators are derived from this class an inherit these attributes.</span>
+
+<span class="sd"> Sensor operators keep executing at a time interval and succeed when</span>
+<span class="sd"> a criteria is met and fail if and when they time out.</span>
+
+<span class="sd"> :param soft_fail: Set to true to mark the task as SKIPPED on failure</span>
+<span class="sd"> :type soft_fail: bool</span>
+<span class="sd"> :param poke_interval: Time in seconds that the job should wait in</span>
+<span class="sd"> between each tries</span>
+<span class="sd"> :type poke_interval: int</span>
+<span class="sd"> :param timeout: Time, in seconds before the task times out and fails.</span>
+<span class="sd"> :type timeout: int</span>
+<span class="sd"> """</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#e6f1f2'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">poke_interval</span><span class="o">=</span><span class="mi">60</span><span class="p">,</span>
+ <span class="n">timeout</span><span class="o">=</span><span class="mi">60</span> <span class="o">*</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">7</span><span class="p">,</span>
+ <span class="n">soft_fail</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">poke_interval</span> <span class="o">=</span> <span class="n">poke_interval</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">soft_fail</span> <span class="o">=</span> <span class="n">soft_fail</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span> <span class="o">=</span> <span class="n">timeout</span>
+
+<div class="viewcode-block" id="BaseSensorOperator.poke"><a class="viewcode-back" href="../../../code.html#airflow.sensors.base_sensor_operator.BaseSensorOperator.poke">[docs]</a> <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Function that the sensors defined while deriving this class should</span>
+<span class="sd"> override.</span>
+<span class="sd"> """</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Override me.'</span><span class="p">)</span></div>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">started_at</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
+ <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">poke</span><span class="p">(</span><span class="n">context</span><span class="p">):</span>
+ <span class="k">if</span> <span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">-</span> <span class="n">started_at</span><span class="p">)</span><span class="o">.</span><span class="n">total_seconds</span><span class="p">()</span> <span class="o">></span> <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">:</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">soft_fail</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowSkipException</span><span class="p">(</span><span class="s1">'Snap. Time is OUT.'</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowSensorTimeout</span><span class="p">(</span><span class="s1">'Snap. Time is OUT.'</span><span class="p">)</span>
+ <span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">poke_interval</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Success criteria met. Exiting."</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/sensors/external_task_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/sensors/external_task_sensor.html b/_modules/airflow/sensors/external_task_sensor.html
new file mode 100644
index 0000000..320aa4c
--- /dev/null
+++ b/_modules/airflow/sensors/external_task_sensor.html
@@ -0,0 +1,336 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.sensors.external_task_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.sensors.external_task_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.sensors.external_task_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">TaskInstance</span>
+<span class="kn">from</span> <span class="nn">airflow.sensors.base_sensor_operator</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.db</span> <span class="k">import</span> <span class="n">provide_session</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span>
+
+
+<div class="viewcode-block" id="ExternalTaskSensor"><a class="viewcode-back" href="../../../code.html#airflow.sensors.external_task_sensor.ExternalTaskSensor">[docs]</a><span class="k">class</span> <span class="nc">ExternalTaskSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Waits for a task to complete in a different DAG</span>
+
+<span class="sd"> :param external_dag_id: The dag_id that contains the task you want to</span>
+<span class="sd"> wait for</span>
+<span class="sd"> :type external_dag_id: string</span>
+<span class="sd"> :param external_task_id: The task_id that contains the task you want to</span>
+<span class="sd"> wait for</span>
+<span class="sd"> :type external_task_id: string</span>
+<span class="sd"> :param allowed_states: list of allowed states, default is ``['success']``</span>
+<span class="sd"> :type allowed_states: list</span>
+<span class="sd"> :param execution_delta: time difference with the previous execution to</span>
+<span class="sd"> look at, the default is the same execution_date as the current task.</span>
+<span class="sd"> For yesterday, use [positive!] datetime.timedelta(days=1). Either</span>
+<span class="sd"> execution_delta or execution_date_fn can be passed to</span>
+<span class="sd"> ExternalTaskSensor, but not both.</span>
+<span class="sd"> :type execution_delta: datetime.timedelta</span>
+<span class="sd"> :param execution_date_fn: function that receives the current execution date</span>
+<span class="sd"> and returns the desired execution dates to query. Either execution_delta</span>
+<span class="sd"> or execution_date_fn can be passed to ExternalTaskSensor, but not both.</span>
+<span class="sd"> :type execution_date_fn: callable</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'external_dag_id'</span><span class="p">,</span> <span class="s1">'external_task_id'</span><span class="p">]</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#19647e'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">external_dag_id</span><span class="p">,</span>
+ <span class="n">external_task_id</span><span class="p">,</span>
+ <span class="n">allowed_states</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">execution_delta</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">execution_date_fn</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">ExternalTaskSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">allowed_states</span> <span class="o">=</span> <span class="n">allowed_states</span> <span class="ow">or</span> <span class="p">[</span><span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">]</span>
+ <span class="k">if</span> <span class="n">execution_delta</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">execution_date_fn</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+ <span class="s1">'Only one of `execution_date` or `execution_date_fn` may'</span>
+ <span class="s1">'be provided to ExternalTaskSensor; not both.'</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">execution_delta</span> <span class="o">=</span> <span class="n">execution_delta</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">execution_date_fn</span> <span class="o">=</span> <span class="n">execution_date_fn</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">external_dag_id</span> <span class="o">=</span> <span class="n">external_dag_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">external_task_id</span> <span class="o">=</span> <span class="n">external_task_id</span>
+
+ <span class="nd">@provide_session</span>
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_delta</span><span class="p">:</span>
+ <span class="n">dttm</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">'execution_date'</span><span class="p">]</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_delta</span>
+ <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date_fn</span><span class="p">:</span>
+ <span class="n">dttm</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date_fn</span><span class="p">(</span><span class="n">context</span><span class="p">[</span><span class="s1">'execution_date'</span><span class="p">])</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">dttm</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">'execution_date'</span><span class="p">]</span>
+
+ <span class="n">dttm_filter</span> <span class="o">=</span> <span class="n">dttm</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dttm</span><span class="p">,</span> <span class="nb">list</span><span class="p">)</span> <span class="k">else</span> <span class="p">[</span><span class="n">dttm</span><span class="p">]</span>
+ <span class="n">serialized_dttm_filter</span> <span class="o">=</span> <span class="s1">','</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
+ <span class="p">[</span><span class="n">datetime</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span> <span class="k">for</span> <span class="n">datetime</span> <span class="ow">in</span> <span class="n">dttm_filter</span><span class="p">])</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'Poking for '</span>
+ <span class="s1">'</span><span class="si">{self.external_dag_id}</span><span class="s1">.'</span>
+ <span class="s1">'</span><span class="si">{self.external_task_id}</span><span class="s1"> on '</span>
+ <span class="s1">'</span><span class="si">{}</span><span class="s1"> ... '</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">serialized_dttm_filter</span><span class="p">,</span> <span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
+ <span class="n">TI</span> <span class="o">=</span> <span class="n">TaskInstance</span>
+
+ <span class="n">count</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TI</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
+ <span class="n">TI</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">external_dag_id</span><span class="p">,</span>
+ <span class="n">TI</span><span class="o">.</span><span class="n">task_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">external_task_id</span><span class="p">,</span>
+ <span class="n">TI</span><span class="o">.</span><span class="n">state</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">allowed_states</span><span class="p">),</span>
+ <span class="n">TI</span><span class="o">.</span><span class="n">execution_date</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">dttm_filter</span><span class="p">),</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">count</span><span class="p">()</span>
+ <span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">count</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">dttm_filter</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[31/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/file_to_gcs.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/file_to_gcs.html b/_modules/airflow/contrib/operators/file_to_gcs.html
new file mode 100644
index 0000000..1be0683
--- /dev/null
+++ b/_modules/airflow/contrib/operators/file_to_gcs.html
@@ -0,0 +1,310 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.file_to_gcs — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.file_to_gcs</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.file_to_gcs</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcs_hook</span> <span class="k">import</span> <span class="n">GoogleCloudStorageHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="FileToGoogleCloudStorageOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.file_to_gcs.FileToGoogleCloudStorageOperator">[docs]</a><span class="k">class</span> <span class="nc">FileToGoogleCloudStorageOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Uploads a file to Google Cloud Storage</span>
+
+<span class="sd"> :param src: Path to the local file</span>
+<span class="sd"> :type src: string</span>
+<span class="sd"> :param dst: Destination path within the specified bucket</span>
+<span class="sd"> :type dst: string</span>
+<span class="sd"> :param bucket: The bucket to upload to</span>
+<span class="sd"> :type bucket: string</span>
+<span class="sd"> :param google_cloud_storage_conn_id: The Airflow connection ID to upload with</span>
+<span class="sd"> :type google_cloud_storage_conn_id: string</span>
+<span class="sd"> :param mime_type: The mime-type string</span>
+<span class="sd"> :type mime_type: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'src'</span><span class="p">,</span> <span class="s1">'dst'</span><span class="p">,</span> <span class="s1">'bucket'</span><span class="p">)</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">src</span><span class="p">,</span>
+ <span class="n">dst</span><span class="p">,</span>
+ <span class="n">bucket</span><span class="p">,</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">mime_type</span><span class="o">=</span><span class="s1">'application/octet-stream'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">FileToGoogleCloudStorageOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">src</span> <span class="o">=</span> <span class="n">src</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">dst</span> <span class="o">=</span> <span class="n">dst</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span> <span class="o">=</span> <span class="n">google_cloud_storage_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mime_type</span> <span class="o">=</span> <span class="n">mime_type</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+
+<div class="viewcode-block" id="FileToGoogleCloudStorageOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.file_to_gcs.FileToGoogleCloudStorageOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Uploads the file to Google cloud storage</span>
+<span class="sd"> """</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+
+ <span class="n">hook</span><span class="o">.</span><span class="n">upload</span><span class="p">(</span>
+ <span class="n">bucket</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span>
+ <span class="nb">object</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dst</span><span class="p">,</span>
+ <span class="n">mime_type</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mime_type</span><span class="p">,</span>
+ <span class="n">filename</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">src</span><span class="p">)</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/file_to_wasb.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/file_to_wasb.html b/_modules/airflow/contrib/operators/file_to_wasb.html
index 0c67052..7e8c93b 100644
--- a/_modules/airflow/contrib/operators/file_to_wasb.html
+++ b/_modules/airflow/contrib/operators/file_to_wasb.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,22 @@
<h1>Source code for airflow.contrib.operators.file_to_wasb</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="c1">#</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.wasb_hook</span> <span class="k">import</span> <span class="n">WasbHook</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
@@ -217,13 +224,13 @@
<span class="bp">self</span><span class="o">.</span><span class="n">wasb_conn_id</span> <span class="o">=</span> <span class="n">wasb_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">load_options</span> <span class="o">=</span> <span class="n">load_options</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="FileToWasbOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.file_to_wasb.FileToWasbOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="sd">"""Upload a file to Azure Blob Storage."""</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">WasbHook</span><span class="p">(</span><span class="n">wasb_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">wasb_conn_id</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
<span class="s1">'Uploading </span><span class="si">{self.file_path}</span><span class="s1"> to wasb://</span><span class="si">{self.container_name}</span><span class="s1"> as </span><span class="si">{self.blob_name}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
<span class="p">)</span>
- <span class="n">hook</span><span class="o">.</span><span class="n">load_file</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">file_path</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">container_name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">blob_name</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">load_options</span><span class="p">)</span></div>
+ <span class="n">hook</span><span class="o">.</span><span class="n">load_file</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">file_path</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">container_name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">blob_name</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">load_options</span><span class="p">)</span></div></div>
</pre></div>
</div>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/gcs_download_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/gcs_download_operator.html b/_modules/airflow/contrib/operators/gcs_download_operator.html
index de261e7..c313433 100644
--- a/_modules/airflow/contrib/operators/gcs_download_operator.html
+++ b/_modules/airflow/contrib/operators/gcs_download_operator.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,22 @@
<h1>Source code for airflow.contrib.operators.gcs_download_operator</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="kn">import</span> <span class="nn">sys</span>
@@ -199,12 +206,12 @@
<span class="sd"> :type object: string</span>
<span class="sd"> :param filename: The file path on the local file system (where the</span>
<span class="sd"> operator is being executed) that the file should be downloaded to.</span>
-<span class="sd"> If false, the downloaded data will not be stored on the local file</span>
+<span class="sd"> If no filename passed, the downloaded data will not be stored on the local file</span>
<span class="sd"> system.</span>
<span class="sd"> :type filename: string</span>
<span class="sd"> :param store_to_xcom_key: If this param is set, the operator will push</span>
<span class="sd"> the contents of the downloaded file to XCom with the key set in this</span>
-<span class="sd"> parameter. If false, the downloaded data will not be pushed to XCom.</span>
+<span class="sd"> parameter. If not set, the downloaded data will not be pushed to XCom.</span>
<span class="sd"> :type store_to_xcom_key: string</span>
<span class="sd"> :param google_cloud_storage_conn_id: The connection ID to use when</span>
<span class="sd"> connecting to Google cloud storage.</span>
@@ -220,9 +227,9 @@
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
<span class="n">bucket</span><span class="p">,</span>
<span class="nb">object</span><span class="p">,</span>
- <span class="n">filename</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">store_to_xcom_key</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_storage_default'</span><span class="p">,</span>
+ <span class="n">filename</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">store_to_xcom_key</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span>
<span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
@@ -235,16 +242,21 @@
<span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing download: </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">object</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">)</span>
- <span class="n">hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span><span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span>
- <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
- <span class="n">file_bytes</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">download</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">object</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing download: </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">object</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">)</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span>
+ <span class="p">)</span>
+ <span class="n">file_bytes</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">download</span><span class="p">(</span><span class="n">bucket</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span>
+ <span class="nb">object</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">object</span><span class="p">,</span>
+ <span class="n">filename</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">store_to_xcom_key</span><span class="p">:</span>
<span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">getsizeof</span><span class="p">(</span><span class="n">file_bytes</span><span class="p">)</span> <span class="o"><</span> <span class="mi">48000</span><span class="p">:</span>
<span class="n">context</span><span class="p">[</span><span class="s1">'ti'</span><span class="p">]</span><span class="o">.</span><span class="n">xcom_push</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">store_to_xcom_key</span><span class="p">,</span> <span class="n">value</span><span class="o">=</span><span class="n">file_bytes</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s1">'The size of the downloaded file is too large to push to XCom!'</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">file_bytes</span><span class="p">)</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="n">file_bytes</span><span class="p">)</span></div>
</pre></div>
</div>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/gcs_list_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/gcs_list_operator.html b/_modules/airflow/contrib/operators/gcs_list_operator.html
new file mode 100644
index 0000000..21013de
--- /dev/null
+++ b/_modules/airflow/contrib/operators/gcs_list_operator.html
@@ -0,0 +1,326 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.gcs_list_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.gcs_list_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.gcs_list_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcs_hook</span> <span class="k">import</span> <span class="n">GoogleCloudStorageHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="GoogleCloudStorageListOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcs_list_operator.GoogleCloudStorageListOperator">[docs]</a><span class="k">class</span> <span class="nc">GoogleCloudStorageListOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> List all objects from the bucket with the give string prefix and delimiter in name.</span>
+
+<span class="sd"> This operator returns a python list with the name of objects which can be used by</span>
+<span class="sd"> `xcom` in the downstream task.</span>
+
+<span class="sd"> :param bucket: The Google cloud storage bucket to find the objects.</span>
+<span class="sd"> :type bucket: string</span>
+<span class="sd"> :param prefix: Prefix string which filters objects whose name begin with this prefix</span>
+<span class="sd"> :type prefix: string</span>
+<span class="sd"> :param delimiter: The delimiter by which you want to filter the objects.</span>
+<span class="sd"> For e.g to lists the CSV files from in a directory in GCS you would use</span>
+<span class="sd"> delimiter='.csv'.</span>
+<span class="sd"> :type delimiter: string</span>
+<span class="sd"> :param google_cloud_storage_conn_id: The connection ID to use when</span>
+<span class="sd"> connecting to Google cloud storage.</span>
+<span class="sd"> :type google_cloud_storage_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request must have</span>
+<span class="sd"> domain-wide delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+
+<span class="sd"> **Example**:</span>
+<span class="sd"> The following Operator would list all the Avro files from ``sales/sales-2017``</span>
+<span class="sd"> folder in ``data`` bucket. ::</span>
+
+<span class="sd"> GCS_Files = GoogleCloudStorageListOperator(</span>
+<span class="sd"> task_id='GCS_Files',</span>
+<span class="sd"> bucket='data',</span>
+<span class="sd"> prefix='sales/sales-2017/',</span>
+<span class="sd"> delimiter='.avro',</span>
+<span class="sd"> google_cloud_storage_conn_id=google_cloud_conn_id</span>
+<span class="sd"> )</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bucket'</span><span class="p">,</span> <span class="s1">'prefix'</span><span class="p">,</span> <span class="s1">'delimiter'</span><span class="p">)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f0eee4'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">bucket</span><span class="p">,</span>
+ <span class="n">prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">delimiter</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">GoogleCloudStorageListOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">prefix</span> <span class="o">=</span> <span class="n">prefix</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delimiter</span> <span class="o">=</span> <span class="n">delimiter</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span> <span class="o">=</span> <span class="n">google_cloud_storage_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span>
+ <span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Getting list of the files. Bucket: </span><span class="si">%s</span><span class="s1">; Delimiter: </span><span class="si">%s</span><span class="s1">; Prefix: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">delimiter</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">prefix</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">list</span><span class="p">(</span><span class="n">bucket</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span>
+ <span class="n">prefix</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">prefix</span><span class="p">,</span>
+ <span class="n">delimiter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delimiter</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/gcs_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/gcs_operator.html b/_modules/airflow/contrib/operators/gcs_operator.html
new file mode 100644
index 0000000..1d95eae
--- /dev/null
+++ b/_modules/airflow/contrib/operators/gcs_operator.html
@@ -0,0 +1,357 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.gcs_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.gcs_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.gcs_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcs_hook</span> <span class="k">import</span> <span class="n">GoogleCloudStorageHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.version</span> <span class="k">import</span> <span class="n">version</span>
+
+
+<div class="viewcode-block" id="GoogleCloudStorageCreateBucketOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcs_operator.GoogleCloudStorageCreateBucketOperator">[docs]</a><span class="k">class</span> <span class="nc">GoogleCloudStorageCreateBucketOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Creates a new bucket. Google Cloud Storage uses a flat namespace,</span>
+<span class="sd"> so you can't create a bucket with a name that is already in use.</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> For more information, see Bucket Naming Guidelines:</span>
+<span class="sd"> https://cloud.google.com/storage/docs/bucketnaming.html#requirements</span>
+
+<span class="sd"> :param bucket_name: The name of the bucket.</span>
+<span class="sd"> :type bucket_name: string</span>
+<span class="sd"> :param storage_class: This defines how objects in the bucket are stored</span>
+<span class="sd"> and determines the SLA and the cost of storage. Values include</span>
+
+<span class="sd"> - ``MULTI_REGIONAL``</span>
+<span class="sd"> - ``REGIONAL``</span>
+<span class="sd"> - ``STANDARD``</span>
+<span class="sd"> - ``NEARLINE``</span>
+<span class="sd"> - ``COLDLINE``.</span>
+<span class="sd"> If this value is not specified when the bucket is</span>
+<span class="sd"> created, it will default to STANDARD.</span>
+<span class="sd"> :type storage_class: string</span>
+<span class="sd"> :param location: The location of the bucket.</span>
+<span class="sd"> Object data for objects in the bucket resides in physical storage</span>
+<span class="sd"> within this region. Defaults to US.</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> https://developers.google.com/storage/docs/bucket-locations</span>
+
+<span class="sd"> :type location: string</span>
+<span class="sd"> :param project_id: The ID of the GCP Project.</span>
+<span class="sd"> :type project_id: string</span>
+<span class="sd"> :param labels: User-provided labels, in key/value pairs.</span>
+<span class="sd"> :type labels: dict</span>
+<span class="sd"> :param google_cloud_storage_conn_id: The connection ID to use when</span>
+<span class="sd"> connecting to Google cloud storage.</span>
+<span class="sd"> :type google_cloud_storage_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request must</span>
+<span class="sd"> have domain-wide delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+
+<span class="sd"> **Example**:</span>
+<span class="sd"> The following Operator would create a new bucket ``test-bucket``</span>
+<span class="sd"> with ``MULTI_REGIONAL`` storage class in ``EU`` region ::</span>
+
+<span class="sd"> CreateBucket = GoogleCloudStorageCreateBucketOperator(</span>
+<span class="sd"> task_id='CreateNewBucket',</span>
+<span class="sd"> bucket_name='test-bucket',</span>
+<span class="sd"> storage_class='MULTI_REGIONAL',</span>
+<span class="sd"> location='EU',</span>
+<span class="sd"> labels={'env': 'dev', 'team': 'airflow'},</span>
+<span class="sd"> google_cloud_storage_conn_id='airflow-service-account'</span>
+<span class="sd"> )</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bucket_name'</span><span class="p">,</span> <span class="s1">'storage_class'</span><span class="p">,</span>
+ <span class="s1">'location'</span><span class="p">,</span> <span class="s1">'project_id'</span><span class="p">)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f0eee4'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">bucket_name</span><span class="p">,</span>
+ <span class="n">storage_class</span><span class="o">=</span><span class="s1">'MULTI_REGIONAL'</span><span class="p">,</span>
+ <span class="n">location</span><span class="o">=</span><span class="s1">'US'</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">GoogleCloudStorageCreateBucketOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bucket_name</span> <span class="o">=</span> <span class="n">bucket_name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">storage_class</span> <span class="o">=</span> <span class="n">storage_class</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">location</span> <span class="o">=</span> <span class="n">location</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="o">=</span> <span class="n">labels</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span> <span class="o">=</span> <span class="n">google_cloud_storage_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">labels</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
+ <span class="p">{</span><span class="s1">'airflow-version'</span><span class="p">:</span> <span class="s1">'v'</span> <span class="o">+</span> <span class="n">version</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'.'</span><span class="p">,</span> <span class="s1">'-'</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'+'</span><span class="p">,</span> <span class="s1">'-'</span><span class="p">)}</span>
+ <span class="p">)</span>
+
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span>
+ <span class="p">)</span>
+
+ <span class="n">hook</span><span class="o">.</span><span class="n">create_bucket</span><span class="p">(</span><span class="n">bucket_name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket_name</span><span class="p">,</span>
+ <span class="n">storage_class</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">storage_class</span><span class="p">,</span>
+ <span class="n">location</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">labels</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">labels</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[43/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/gcs_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/gcs_hook.html b/_modules/airflow/contrib/hooks/gcs_hook.html
index 6f3bc49..5db94f8 100644
--- a/_modules/airflow/contrib/hooks/gcs_hook.html
+++ b/_modules/airflow/contrib/hooks/gcs_hook.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -169,23 +171,31 @@
<h1>Source code for airflow.contrib.hooks.gcs_hook</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="c1">#</span>
<span class="kn">from</span> <span class="nn">apiclient.discovery</span> <span class="k">import</span> <span class="n">build</span>
<span class="kn">from</span> <span class="nn">apiclient.http</span> <span class="k">import</span> <span class="n">MediaFileUpload</span>
<span class="kn">from</span> <span class="nn">googleapiclient</span> <span class="k">import</span> <span class="n">errors</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="k">import</span> <span class="n">GoogleCloudBaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+
+<span class="kn">import</span> <span class="nn">re</span>
<div class="viewcode-block" id="GoogleCloudStorageHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook">[docs]</a><span class="k">class</span> <span class="nc">GoogleCloudStorageHook</span><span class="p">(</span><span class="n">GoogleCloudBaseHook</span><span class="p">):</span>
@@ -195,7 +205,7 @@
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
- <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_storage_default'</span><span class="p">,</span>
+ <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">GoogleCloudStorageHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="p">)</span>
@@ -207,7 +217,6 @@
<span class="n">http_authorized</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
<span class="k">return</span> <span class="n">build</span><span class="p">(</span><span class="s1">'storage'</span><span class="p">,</span> <span class="s1">'v1'</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">)</span></div>
-
<span class="c1"># pylint:disable=redefined-builtin</span>
<div class="viewcode-block" id="GoogleCloudStorageHook.copy"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.copy">[docs]</a> <span class="k">def</span> <span class="nf">copy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">source_bucket</span><span class="p">,</span> <span class="n">source_object</span><span class="p">,</span> <span class="n">destination_bucket</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">destination_object</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
@@ -217,10 +226,10 @@
<span class="sd"> destination_bucket or destination_object can be omitted, in which case</span>
<span class="sd"> source bucket/object is used, but not both.</span>
-<span class="sd"> :param bucket: The bucket of the object to copy from.</span>
-<span class="sd"> :type bucket: string</span>
-<span class="sd"> :param object: The object to copy.</span>
-<span class="sd"> :type object: string</span>
+<span class="sd"> :param source_bucket: The bucket of the object to copy from.</span>
+<span class="sd"> :type source_bucket: string</span>
+<span class="sd"> :param source_object: The object to copy.</span>
+<span class="sd"> :type source_object: string</span>
<span class="sd"> :param destination_bucket: The destination of the object to copied to.</span>
<span class="sd"> Can be omitted; then the same bucket is used.</span>
<span class="sd"> :type destination_bucket: string</span>
@@ -252,9 +261,60 @@
<span class="k">return</span> <span class="kc">False</span>
<span class="k">raise</span></div>
+<div class="viewcode-block" id="GoogleCloudStorageHook.rewrite"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.rewrite">[docs]</a> <span class="k">def</span> <span class="nf">rewrite</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">source_bucket</span><span class="p">,</span> <span class="n">source_object</span><span class="p">,</span> <span class="n">destination_bucket</span><span class="p">,</span>
+ <span class="n">destination_object</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Has the same functionality as copy, except that will work on files</span>
+<span class="sd"> over 5 TB, as well as when copying between locations and/or storage</span>
+<span class="sd"> classes.</span>
+
+<span class="sd"> destination_object can be omitted, in which case source_object is used.</span>
+
+<span class="sd"> :param source_bucket: The bucket of the object to copy from.</span>
+<span class="sd"> :type source_bucket: string</span>
+<span class="sd"> :param source_object: The object to copy.</span>
+<span class="sd"> :type source_object: string</span>
+<span class="sd"> :param destination_bucket: The destination of the object to copied to.</span>
+<span class="sd"> :type destination_bucket: string</span>
+<span class="sd"> :param destination_object: The (renamed) path of the object if given.</span>
+<span class="sd"> Can be omitted; then the same name is used.</span>
+<span class="sd"> """</span>
+ <span class="n">destination_object</span> <span class="o">=</span> <span class="n">destination_object</span> <span class="ow">or</span> <span class="n">source_object</span>
+ <span class="k">if</span> <span class="p">(</span><span class="n">source_bucket</span> <span class="o">==</span> <span class="n">destination_bucket</span> <span class="ow">and</span>
+ <span class="n">source_object</span> <span class="o">==</span> <span class="n">destination_object</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+ <span class="s1">'Either source/destination bucket or source/destination object '</span>
+ <span class="s1">'must be different, not both the same: bucket=</span><span class="si">%s</span><span class="s1">, object=</span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span>
+ <span class="p">(</span><span class="n">source_bucket</span><span class="p">,</span> <span class="n">source_object</span><span class="p">))</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">source_bucket</span> <span class="ow">or</span> <span class="ow">not</span> <span class="n">source_object</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'source_bucket and source_object cannot be empty.'</span><span class="p">)</span>
+
+ <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">request_count</span> <span class="o">=</span> <span class="mi">1</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="n">service</span><span class="o">.</span><span class="n">objects</span><span class="p">()</span> \
+ <span class="o">.</span><span class="n">rewrite</span><span class="p">(</span><span class="n">sourceBucket</span><span class="o">=</span><span class="n">source_bucket</span><span class="p">,</span> <span class="n">sourceObject</span><span class="o">=</span><span class="n">source_object</span><span class="p">,</span>
+ <span class="n">destinationBucket</span><span class="o">=</span><span class="n">destination_bucket</span><span class="p">,</span>
+ <span class="n">destinationObject</span><span class="o">=</span><span class="n">destination_object</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="s1">''</span><span class="p">)</span> \
+ <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Rewrite request #</span><span class="si">%s</span><span class="s1">: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">request_count</span><span class="p">,</span> <span class="n">result</span><span class="p">)</span>
+ <span class="k">while</span> <span class="ow">not</span> <span class="n">result</span><span class="p">[</span><span class="s1">'done'</span><span class="p">]:</span>
+ <span class="n">request_count</span> <span class="o">+=</span> <span class="mi">1</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="n">service</span><span class="o">.</span><span class="n">objects</span><span class="p">()</span> \
+ <span class="o">.</span><span class="n">rewrite</span><span class="p">(</span><span class="n">sourceBucket</span><span class="o">=</span><span class="n">source_bucket</span><span class="p">,</span> <span class="n">sourceObject</span><span class="o">=</span><span class="n">source_object</span><span class="p">,</span>
+ <span class="n">destinationBucket</span><span class="o">=</span><span class="n">destination_bucket</span><span class="p">,</span>
+ <span class="n">destinationObject</span><span class="o">=</span><span class="n">destination_object</span><span class="p">,</span>
+ <span class="n">rewriteToken</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s1">'rewriteToken'</span><span class="p">],</span> <span class="n">body</span><span class="o">=</span><span class="s1">''</span><span class="p">)</span> \
+ <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Rewrite request #</span><span class="si">%s</span><span class="s1">: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">request_count</span><span class="p">,</span> <span class="n">result</span><span class="p">)</span>
+ <span class="k">return</span> <span class="kc">True</span>
+ <span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">ex</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">ex</span><span class="o">.</span><span class="n">resp</span><span class="p">[</span><span class="s1">'status'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'404'</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">False</span>
+ <span class="k">raise</span></div>
<span class="c1"># pylint:disable=redefined-builtin</span>
-<div class="viewcode-block" id="GoogleCloudStorageHook.download"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.download">[docs]</a> <span class="k">def</span> <span class="nf">download</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">filename</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+<div class="viewcode-block" id="GoogleCloudStorageHook.download"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.download">[docs]</a> <span class="k">def</span> <span class="nf">download</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">filename</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Get a file from Google Cloud Storage.</span>
@@ -388,7 +448,7 @@
<span class="k">return</span> <span class="kc">False</span>
<span class="k">raise</span></div>
-<div class="viewcode-block" id="GoogleCloudStorageHook.list"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.list">[docs]</a> <span class="k">def</span> <span class="nf">list</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="n">versions</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">maxResults</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+<div class="viewcode-block" id="GoogleCloudStorageHook.list"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.list">[docs]</a> <span class="k">def</span> <span class="nf">list</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="n">versions</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">maxResults</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> List all objects from the bucket with the give string prefix in name</span>
@@ -400,6 +460,8 @@
<span class="sd"> :type maxResults: integer</span>
<span class="sd"> :param prefix: prefix string which filters objects whose name begin with this prefix</span>
<span class="sd"> :type prefix: string</span>
+<span class="sd"> :param delimiter: filters objects based on the delimiter (for e.g '.csv')</span>
+<span class="sd"> :type delimiter: string</span>
<span class="sd"> :return: a stream of object names matching the filtering criteria</span>
<span class="sd"> """</span>
<span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
@@ -412,16 +474,21 @@
<span class="n">versions</span><span class="o">=</span><span class="n">versions</span><span class="p">,</span>
<span class="n">maxResults</span><span class="o">=</span><span class="n">maxResults</span><span class="p">,</span>
<span class="n">pageToken</span><span class="o">=</span><span class="n">pageToken</span><span class="p">,</span>
- <span class="n">prefix</span><span class="o">=</span><span class="n">prefix</span>
+ <span class="n">prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">,</span>
+ <span class="n">delimiter</span><span class="o">=</span><span class="n">delimiter</span>
<span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
- <span class="k">if</span> <span class="s1">'items'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">response</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"No items found for prefix: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">prefix</span><span class="p">)</span>
- <span class="k">break</span>
+ <span class="k">if</span> <span class="s1">'prefixes'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">response</span><span class="p">:</span>
+ <span class="k">if</span> <span class="s1">'items'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">response</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"No items found for prefix: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">prefix</span><span class="p">)</span>
+ <span class="k">break</span>
- <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">response</span><span class="p">[</span><span class="s1">'items'</span><span class="p">]:</span>
- <span class="k">if</span> <span class="n">item</span> <span class="ow">and</span> <span class="s1">'name'</span> <span class="ow">in</span> <span class="n">item</span><span class="p">:</span>
- <span class="n">ids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">[</span><span class="s1">'name'</span><span class="p">])</span>
+ <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">response</span><span class="p">[</span><span class="s1">'items'</span><span class="p">]:</span>
+ <span class="k">if</span> <span class="n">item</span> <span class="ow">and</span> <span class="s1">'name'</span> <span class="ow">in</span> <span class="n">item</span><span class="p">:</span>
+ <span class="n">ids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">[</span><span class="s1">'name'</span><span class="p">])</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">response</span><span class="p">[</span><span class="s1">'prefixes'</span><span class="p">]:</span>
+ <span class="n">ids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="k">if</span> <span class="s1">'nextPageToken'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">response</span><span class="p">:</span>
<span class="c1"># no further pages of results, so stop the loop</span>
@@ -431,7 +498,203 @@
<span class="k">if</span> <span class="ow">not</span> <span class="n">pageToken</span><span class="p">:</span>
<span class="c1"># empty next page token</span>
<span class="k">break</span>
- <span class="k">return</span> <span class="n">ids</span></div></div>
+ <span class="k">return</span> <span class="n">ids</span></div>
+
+<div class="viewcode-block" id="GoogleCloudStorageHook.get_size"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.get_size">[docs]</a> <span class="k">def</span> <span class="nf">get_size</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Gets the size of a file in Google Cloud Storage.</span>
+
+<span class="sd"> :param bucket: The Google cloud storage bucket where the object is.</span>
+<span class="sd"> :type bucket: string</span>
+<span class="sd"> :param object: The name of the object to check in the Google cloud storage bucket.</span>
+<span class="sd"> :type object: string</span>
+
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Checking the file size of object: </span><span class="si">%s</span><span class="s1"> in bucket: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">bucket</span><span class="p">)</span>
+ <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="n">service</span><span class="o">.</span><span class="n">objects</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="n">bucket</span><span class="o">=</span><span class="n">bucket</span><span class="p">,</span>
+ <span class="nb">object</span><span class="o">=</span><span class="nb">object</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+
+ <span class="k">if</span> <span class="s1">'name'</span> <span class="ow">in</span> <span class="n">response</span> <span class="ow">and</span> <span class="n">response</span><span class="p">[</span><span class="s1">'name'</span><span class="p">][</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="o">!=</span> <span class="s1">'/'</span><span class="p">:</span>
+ <span class="c1"># Remove Directories & Just check size of files</span>
+ <span class="n">size</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">'size'</span><span class="p">]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'The file size of </span><span class="si">%s</span><span class="s1"> is </span><span class="si">%s</span><span class="s1"> bytes.'</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">size</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">size</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'Object is not a file'</span><span class="p">)</span>
+ <span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">ex</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">ex</span><span class="o">.</span><span class="n">resp</span><span class="p">[</span><span class="s1">'status'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'404'</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'Object Not Found'</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="GoogleCloudStorageHook.get_crc32c"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.get_crc32c">[docs]</a> <span class="k">def</span> <span class="nf">get_crc32c</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Gets the CRC32c checksum of an object in Google Cloud Storage.</span>
+
+<span class="sd"> :param bucket: The Google cloud storage bucket where the object is.</span>
+<span class="sd"> :type bucket: string</span>
+<span class="sd"> :param object: The name of the object to check in the Google cloud</span>
+<span class="sd"> storage bucket.</span>
+<span class="sd"> :type object: string</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Retrieving the crc32c checksum of '</span>
+ <span class="s1">'object: </span><span class="si">%s</span><span class="s1"> in bucket: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">bucket</span><span class="p">)</span>
+ <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="n">service</span><span class="o">.</span><span class="n">objects</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="n">bucket</span><span class="o">=</span><span class="n">bucket</span><span class="p">,</span>
+ <span class="nb">object</span><span class="o">=</span><span class="nb">object</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+
+ <span class="n">crc32c</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">'crc32c'</span><span class="p">]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'The crc32c checksum of </span><span class="si">%s</span><span class="s1"> is </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">crc32c</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">crc32c</span>
+
+ <span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">ex</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">ex</span><span class="o">.</span><span class="n">resp</span><span class="p">[</span><span class="s1">'status'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'404'</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'Object Not Found'</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="GoogleCloudStorageHook.get_md5hash"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.get_md5hash">[docs]</a> <span class="k">def</span> <span class="nf">get_md5hash</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Gets the MD5 hash of an object in Google Cloud Storage.</span>
+
+<span class="sd"> :param bucket: The Google cloud storage bucket where the object is.</span>
+<span class="sd"> :type bucket: string</span>
+<span class="sd"> :param object: The name of the object to check in the Google cloud</span>
+<span class="sd"> storage bucket.</span>
+<span class="sd"> :type object: string</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Retrieving the MD5 hash of '</span>
+ <span class="s1">'object: </span><span class="si">%s</span><span class="s1"> in bucket: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">bucket</span><span class="p">)</span>
+ <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="n">service</span><span class="o">.</span><span class="n">objects</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="n">bucket</span><span class="o">=</span><span class="n">bucket</span><span class="p">,</span>
+ <span class="nb">object</span><span class="o">=</span><span class="nb">object</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+
+ <span class="n">md5hash</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">'md5Hash'</span><span class="p">]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'The md5Hash of </span><span class="si">%s</span><span class="s1"> is </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">md5hash</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">md5hash</span>
+
+ <span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">ex</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">ex</span><span class="o">.</span><span class="n">resp</span><span class="p">[</span><span class="s1">'status'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'404'</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'Object Not Found'</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="GoogleCloudStorageHook.create_bucket"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.create_bucket">[docs]</a> <span class="k">def</span> <span class="nf">create_bucket</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">bucket_name</span><span class="p">,</span>
+ <span class="n">storage_class</span><span class="o">=</span><span class="s1">'MULTI_REGIONAL'</span><span class="p">,</span>
+ <span class="n">location</span><span class="o">=</span><span class="s1">'US'</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">labels</span><span class="o">=</span><span class="kc">None</span>
+ <span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Creates a new bucket. Google Cloud Storage uses a flat namespace, so</span>
+<span class="sd"> you can't create a bucket with a name that is already in use.</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> For more information, see Bucket Naming Guidelines:</span>
+<span class="sd"> https://cloud.google.com/storage/docs/bucketnaming.html#requirements</span>
+
+<span class="sd"> :param bucket_name: The name of the bucket.</span>
+<span class="sd"> :type bucket_name: string</span>
+<span class="sd"> :param storage_class: This defines how objects in the bucket are stored</span>
+<span class="sd"> and determines the SLA and the cost of storage. Values include</span>
+
+<span class="sd"> - ``MULTI_REGIONAL``</span>
+<span class="sd"> - ``REGIONAL``</span>
+<span class="sd"> - ``STANDARD``</span>
+<span class="sd"> - ``NEARLINE``</span>
+<span class="sd"> - ``COLDLINE``.</span>
+<span class="sd"> If this value is not specified when the bucket is</span>
+<span class="sd"> created, it will default to STANDARD.</span>
+<span class="sd"> :type storage_class: string</span>
+<span class="sd"> :param location: The location of the bucket.</span>
+<span class="sd"> Object data for objects in the bucket resides in physical storage</span>
+<span class="sd"> within this region. Defaults to US.</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> https://developers.google.com/storage/docs/bucket-locations</span>
+
+<span class="sd"> :type location: string</span>
+<span class="sd"> :param project_id: The ID of the GCP Project.</span>
+<span class="sd"> :type project_id: string</span>
+<span class="sd"> :param labels: User-provided labels, in key/value pairs.</span>
+<span class="sd"> :type labels: dict</span>
+<span class="sd"> :return: If successful, it returns the ``id`` of the bucket.</span>
+<span class="sd"> """</span>
+
+ <span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span> <span class="k">if</span> <span class="n">project_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
+ <span class="n">storage_classes</span> <span class="o">=</span> <span class="p">[</span>
+ <span class="s1">'MULTI_REGIONAL'</span><span class="p">,</span>
+ <span class="s1">'REGIONAL'</span><span class="p">,</span>
+ <span class="s1">'NEARLINE'</span><span class="p">,</span>
+ <span class="s1">'COLDLINE'</span><span class="p">,</span>
+ <span class="s1">'STANDARD'</span><span class="p">,</span> <span class="c1"># alias for MULTI_REGIONAL/REGIONAL, based on location</span>
+ <span class="p">]</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Creating Bucket: </span><span class="si">%s</span><span class="s1">; Location: </span><span class="si">%s</span><span class="s1">; Storage Class: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">bucket_name</span><span class="p">,</span> <span class="n">location</span><span class="p">,</span> <span class="n">storage_class</span><span class="p">)</span>
+ <span class="k">assert</span> <span class="n">storage_class</span> <span class="ow">in</span> <span class="n">storage_classes</span><span class="p">,</span> \
+ <span class="s1">'Invalid value (</span><span class="si">{}</span><span class="s1">) passed to storage_class. Value should be '</span> \
+ <span class="s1">'one of </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">storage_class</span><span class="p">,</span> <span class="n">storage_classes</span><span class="p">)</span>
+
+ <span class="k">assert</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="s1">'[a-zA-Z0-9]+'</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">[</span><span class="mi">0</span><span class="p">]),</span> \
+ <span class="s1">'Bucket names must start with a number or letter.'</span>
+
+ <span class="k">assert</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="s1">'[a-zA-Z0-9]+'</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]),</span> \
+ <span class="s1">'Bucket names must end with a number or letter.'</span>
+
+ <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">bucket_resource</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'name'</span><span class="p">:</span> <span class="n">bucket_name</span><span class="p">,</span>
+ <span class="s1">'location'</span><span class="p">:</span> <span class="n">location</span><span class="p">,</span>
+ <span class="s1">'storageClass'</span><span class="p">:</span> <span class="n">storage_class</span>
+ <span class="p">}</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'The Default Project ID is </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">labels</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">bucket_resource</span><span class="p">[</span><span class="s1">'labels'</span><span class="p">]</span> <span class="o">=</span> <span class="n">labels</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="n">service</span><span class="o">.</span><span class="n">buckets</span><span class="p">()</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span>
+ <span class="n">project</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">body</span><span class="o">=</span><span class="n">bucket_resource</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Bucket: </span><span class="si">%s</span><span class="s1"> created successfully.'</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">response</span><span class="p">[</span><span class="s1">'id'</span><span class="p">]</span>
+
+ <span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">ex</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'Bucket creation failed. Error was: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">ex</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
+ <span class="p">)</span></div></div>
+
+
+<span class="k">def</span> <span class="nf">_parse_gcs_url</span><span class="p">(</span><span class="n">gsurl</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Given a Google Cloud Storage URL (gs://<bucket>/<blob>), returns a</span>
+<span class="sd"> tuple containing the corresponding bucket and blob.</span>
+<span class="sd"> """</span>
+ <span class="c1"># Python 3</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="k">import</span> <span class="n">urlparse</span>
+ <span class="c1"># Python 2</span>
+ <span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
+ <span class="kn">from</span> <span class="nn">urlparse</span> <span class="k">import</span> <span class="n">urlparse</span>
+
+ <span class="n">parsed_url</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">gsurl</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Please provide a bucket name'</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">bucket</span> <span class="o">=</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span>
+ <span class="c1"># Remove leading '/' but NOT trailing one</span>
+ <span class="n">blob</span> <span class="o">=</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s1">'/'</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">bucket</span><span class="p">,</span> <span class="n">blob</span>
</pre></div>
</div>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/jenkins_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/jenkins_hook.html b/_modules/airflow/contrib/hooks/jenkins_hook.html
new file mode 100644
index 0000000..783bfa6
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/jenkins_hook.html
@@ -0,0 +1,283 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.jenkins_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.jenkins_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.jenkins_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+
+<span class="kn">import</span> <span class="nn">jenkins</span>
+<span class="kn">import</span> <span class="nn">distutils</span>
+
+
+<div class="viewcode-block" id="JenkinsHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.jenkins_hook.JenkinsHook">[docs]</a><span class="k">class</span> <span class="nc">JenkinsHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Hook to manage connection to jenkins server</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn_id</span><span class="o">=</span><span class="s1">'jenkins_default'</span><span class="p">):</span>
+ <span class="n">connection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">conn_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">connection</span> <span class="o">=</span> <span class="n">connection</span>
+ <span class="n">connectionPrefix</span> <span class="o">=</span> <span class="s1">'http'</span>
+ <span class="c1"># connection.extra contains info about using https (true) or http (false)</span>
+ <span class="k">if</span> <span class="n">connection</span><span class="o">.</span><span class="n">extra</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">connection</span><span class="o">.</span><span class="n">extra</span> <span class="o">==</span> <span class="s1">''</span><span class="p">:</span>
+ <span class="n">connection</span><span class="o">.</span><span class="n">extra</span> <span class="o">=</span> <span class="s1">'false'</span>
+ <span class="c1"># set a default value to connection.extra</span>
+ <span class="c1"># to avoid rising ValueError in strtobool</span>
+ <span class="k">if</span> <span class="n">distutils</span><span class="o">.</span><span class="n">util</span><span class="o">.</span><span class="n">strtobool</span><span class="p">(</span><span class="n">connection</span><span class="o">.</span><span class="n">extra</span><span class="p">):</span>
+ <span class="n">connectionPrefix</span> <span class="o">=</span> <span class="s1">'https'</span>
+ <span class="n">url</span> <span class="o">=</span> <span class="s1">'</span><span class="si">%s</span><span class="s1">://</span><span class="si">%s</span><span class="s1">:</span><span class="si">%d</span><span class="s1">'</span> <span class="o">%</span> <span class="p">(</span><span class="n">connectionPrefix</span><span class="p">,</span> <span class="n">connection</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> <span class="n">connection</span><span class="o">.</span><span class="n">port</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Trying to connect to </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">jenkins_server</span> <span class="o">=</span> <span class="n">jenkins</span><span class="o">.</span><span class="n">Jenkins</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">connection</span><span class="o">.</span><span class="n">login</span><span class="p">,</span> <span class="n">connection</span><span class="o">.</span><span class="n">password</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">get_jenkins_server</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">jenkins_server</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/jira_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/jira_hook.html b/_modules/airflow/contrib/hooks/jira_hook.html
new file mode 100644
index 0000000..7dc6dc0
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/jira_hook.html
@@ -0,0 +1,319 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.jira_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.jira_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.jira_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">from</span> <span class="nn">jira</span> <span class="k">import</span> <span class="n">JIRA</span>
+<span class="kn">from</span> <span class="nn">jira.exceptions</span> <span class="k">import</span> <span class="n">JIRAError</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+
+<div class="viewcode-block" id="JiraHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.jira_hook.JiraHook">[docs]</a><span class="k">class</span> <span class="nc">JiraHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Jira interaction hook, a Wrapper around JIRA Python SDK.</span>
+
+<span class="sd"> :param jira_conn_id: reference to a pre-defined Jira Connection</span>
+<span class="sd"> :type jira_conn_id: string</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">jira_conn_id</span><span class="o">=</span><span class="s1">'jira_default'</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">JiraHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">jira_conn_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">jira_conn_id</span> <span class="o">=</span> <span class="n">jira_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+ <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'Creating Jira client for conn_id: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">jira_conn_id</span><span class="p">)</span>
+
+ <span class="n">get_server_info</span> <span class="o">=</span> <span class="kc">True</span>
+ <span class="n">validate</span> <span class="o">=</span> <span class="kc">True</span>
+ <span class="n">extra_options</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="kc">None</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">jira_conn_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">jira_conn_id</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">extra_options</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span>
+ <span class="c1"># only required attributes are taken for now,</span>
+ <span class="c1"># more can be added ex: async, logging, max_retries</span>
+
+ <span class="c1"># verify</span>
+ <span class="k">if</span> <span class="s1">'verify'</span> <span class="ow">in</span> <span class="n">extra_options</span> \
+ <span class="ow">and</span> <span class="n">extra_options</span><span class="p">[</span><span class="s1">'verify'</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'false'</span><span class="p">:</span>
+ <span class="n">extra_options</span><span class="p">[</span><span class="s1">'verify'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
+
+ <span class="c1"># validate</span>
+ <span class="k">if</span> <span class="s1">'validate'</span> <span class="ow">in</span> <span class="n">extra_options</span> \
+ <span class="ow">and</span> <span class="n">extra_options</span><span class="p">[</span><span class="s1">'validate'</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'false'</span><span class="p">:</span>
+ <span class="n">validate</span> <span class="o">=</span> <span class="kc">False</span>
+
+ <span class="k">if</span> <span class="s1">'get_server_info'</span> <span class="ow">in</span> <span class="n">extra_options</span> \
+ <span class="ow">and</span> <span class="n">extra_options</span><span class="p">[</span><span class="s1">'get_server_info'</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'false'</span><span class="p">:</span>
+ <span class="n">get_server_info</span> <span class="o">=</span> <span class="kc">False</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="n">JIRA</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">,</span>
+ <span class="n">options</span><span class="o">=</span><span class="n">extra_options</span><span class="p">,</span>
+ <span class="n">basic_auth</span><span class="o">=</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">),</span>
+ <span class="n">get_server_info</span><span class="o">=</span><span class="n">get_server_info</span><span class="p">,</span>
+ <span class="n">validate</span><span class="o">=</span><span class="n">validate</span><span class="p">)</span>
+ <span class="k">except</span> <span class="n">JIRAError</span> <span class="k">as</span> <span class="n">jira_error</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Failed to create jira client, jira error: </span><span class="si">%s</span><span class="s1">'</span>
+ <span class="o">%</span> <span class="nb">str</span><span class="p">(</span><span class="n">jira_error</span><span class="p">))</span>
+ <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Failed to create jira client, error: </span><span class="si">%s</span><span class="s1">'</span>
+ <span class="o">%</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
+
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[08/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/python_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/python_operator.html b/_modules/airflow/operators/python_operator.html
new file mode 100644
index 0000000..fb1e29d
--- /dev/null
+++ b/_modules/airflow/operators/python_operator.html
@@ -0,0 +1,613 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.operators.python_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.operators.python_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.operators.python_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
+<span class="kn">import</span> <span class="nn">dill</span>
+<span class="kn">import</span> <span class="nn">inspect</span>
+<span class="kn">import</span> <span class="nn">os</span>
+<span class="kn">import</span> <span class="nn">pickle</span>
+<span class="kn">import</span> <span class="nn">subprocess</span>
+<span class="kn">import</span> <span class="nn">sys</span>
+<span class="kn">import</span> <span class="nn">types</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span><span class="p">,</span> <span class="n">SkipMixin</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.file</span> <span class="k">import</span> <span class="n">TemporaryDirectory</span>
+
+<span class="kn">from</span> <span class="nn">textwrap</span> <span class="k">import</span> <span class="n">dedent</span>
+
+
+<div class="viewcode-block" id="PythonOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.python_operator.PythonOperator">[docs]</a><span class="k">class</span> <span class="nc">PythonOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Executes a Python callable</span>
+
+<span class="sd"> :param python_callable: A reference to an object that is callable</span>
+<span class="sd"> :type python_callable: python callable</span>
+<span class="sd"> :param op_kwargs: a dictionary of keyword arguments that will get unpacked</span>
+<span class="sd"> in your function</span>
+<span class="sd"> :type op_kwargs: dict</span>
+<span class="sd"> :param op_args: a list of positional arguments that will get unpacked when</span>
+<span class="sd"> calling your callable</span>
+<span class="sd"> :type op_args: list</span>
+<span class="sd"> :param provide_context: if set to true, Airflow will pass a set of</span>
+<span class="sd"> keyword arguments that can be used in your function. This set of</span>
+<span class="sd"> kwargs correspond exactly to what you can use in your jinja</span>
+<span class="sd"> templates. For this to work, you need to define `**kwargs` in your</span>
+<span class="sd"> function header.</span>
+<span class="sd"> :type provide_context: bool</span>
+<span class="sd"> :param templates_dict: a dictionary where the values are templates that</span>
+<span class="sd"> will get templated by the Airflow engine sometime between</span>
+<span class="sd"> ``__init__`` and ``execute`` takes place and are made available</span>
+<span class="sd"> in your callable's context after the template has been applied</span>
+<span class="sd"> :type templates_dict: dict of str</span>
+<span class="sd"> :param templates_exts: a list of file extensions to resolve while</span>
+<span class="sd"> processing templated fields, for examples ``['.sql', '.hql']``</span>
+<span class="sd"> :type templates_exts: list(str)</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'templates_dict'</span><span class="p">,)</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">()</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#ffefeb'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">python_callable</span><span class="p">,</span>
+ <span class="n">op_args</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">op_kwargs</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">provide_context</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">templates_dict</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">templates_exts</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">PythonOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">callable</span><span class="p">(</span><span class="n">python_callable</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'`python_callable` param must be callable'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">python_callable</span> <span class="o">=</span> <span class="n">python_callable</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">op_args</span> <span class="o">=</span> <span class="n">op_args</span> <span class="ow">or</span> <span class="p">[]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">op_kwargs</span> <span class="o">=</span> <span class="n">op_kwargs</span> <span class="ow">or</span> <span class="p">{}</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">provide_context</span> <span class="o">=</span> <span class="n">provide_context</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">templates_dict</span> <span class="o">=</span> <span class="n">templates_dict</span>
+ <span class="k">if</span> <span class="n">templates_exts</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">template_ext</span> <span class="o">=</span> <span class="n">templates_exts</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">provide_context</span><span class="p">:</span>
+ <span class="n">context</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">op_kwargs</span><span class="p">)</span>
+ <span class="n">context</span><span class="p">[</span><span class="s1">'templates_dict'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">templates_dict</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">op_kwargs</span> <span class="o">=</span> <span class="n">context</span>
+
+ <span class="n">return_value</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">execute_callable</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Done. Returned value was: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">return_value</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">return_value</span>
+
+ <span class="k">def</span> <span class="nf">execute_callable</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">python_callable</span><span class="p">(</span><span class="o">*</span><span class="bp">self</span><span class="o">.</span><span class="n">op_args</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">op_kwargs</span><span class="p">)</span></div>
+
+
+<div class="viewcode-block" id="BranchPythonOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.python_operator.BranchPythonOperator">[docs]</a><span class="k">class</span> <span class="nc">BranchPythonOperator</span><span class="p">(</span><span class="n">PythonOperator</span><span class="p">,</span> <span class="n">SkipMixin</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Allows a workflow to "branch" or follow a single path following the</span>
+<span class="sd"> execution of this task.</span>
+
+<span class="sd"> It derives the PythonOperator and expects a Python function that returns</span>
+<span class="sd"> the task_id to follow. The task_id returned should point to a task</span>
+<span class="sd"> directly downstream from {self}. All other "branches" or</span>
+<span class="sd"> directly downstream tasks are marked with a state of ``skipped`` so that</span>
+<span class="sd"> these paths can't move forward. The ``skipped`` states are propageted</span>
+<span class="sd"> downstream to allow for the DAG state to fill up and the DAG run's state</span>
+<span class="sd"> to be inferred.</span>
+
+<span class="sd"> Note that using tasks with ``depends_on_past=True`` downstream from</span>
+<span class="sd"> ``BranchPythonOperator`` is logically unsound as ``skipped`` status</span>
+<span class="sd"> will invariably lead to block tasks that depend on their past successes.</span>
+<span class="sd"> ``skipped`` states propagates where all directly upstream tasks are</span>
+<span class="sd"> ``skipped``.</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">branch</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">BranchPythonOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">context</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Following branch </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">branch</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Marking other directly downstream tasks as skipped"</span><span class="p">)</span>
+
+ <span class="n">downstream_tasks</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">'task'</span><span class="p">]</span><span class="o">.</span><span class="n">downstream_list</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Downstream task_ids </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">downstream_tasks</span><span class="p">)</span>
+
+ <span class="n">skip_tasks</span> <span class="o">=</span> <span class="p">[</span><span class="n">t</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">downstream_tasks</span> <span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">task_id</span> <span class="o">!=</span> <span class="n">branch</span><span class="p">]</span>
+ <span class="k">if</span> <span class="n">downstream_tasks</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">skip</span><span class="p">(</span><span class="n">context</span><span class="p">[</span><span class="s1">'dag_run'</span><span class="p">],</span> <span class="n">context</span><span class="p">[</span><span class="s1">'ti'</span><span class="p">]</span><span class="o">.</span><span class="n">execution_date</span><span class="p">,</span> <span class="n">skip_tasks</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Done."</span><span class="p">)</span></div>
+
+
+<div class="viewcode-block" id="ShortCircuitOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.python_operator.ShortCircuitOperator">[docs]</a><span class="k">class</span> <span class="nc">ShortCircuitOperator</span><span class="p">(</span><span class="n">PythonOperator</span><span class="p">,</span> <span class="n">SkipMixin</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Allows a workflow to continue only if a condition is met. Otherwise, the</span>
+<span class="sd"> workflow "short-circuits" and downstream tasks are skipped.</span>
+
+<span class="sd"> The ShortCircuitOperator is derived from the PythonOperator. It evaluates a</span>
+<span class="sd"> condition and short-circuits the workflow if the condition is False. Any</span>
+<span class="sd"> downstream tasks are marked with a state of "skipped". If the condition is</span>
+<span class="sd"> True, downstream tasks proceed as normal.</span>
+
+<span class="sd"> The condition is determined by the result of `python_callable`.</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">condition</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">ShortCircuitOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">context</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Condition result is </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">condition</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">condition</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Proceeding with downstream tasks...'</span><span class="p">)</span>
+ <span class="k">return</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Skipping downstream tasks...'</span><span class="p">)</span>
+
+ <span class="n">downstream_tasks</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">'task'</span><span class="p">]</span><span class="o">.</span><span class="n">get_flat_relatives</span><span class="p">(</span><span class="n">upstream</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Downstream task_ids </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">downstream_tasks</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">downstream_tasks</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">skip</span><span class="p">(</span><span class="n">context</span><span class="p">[</span><span class="s1">'dag_run'</span><span class="p">],</span> <span class="n">context</span><span class="p">[</span><span class="s1">'ti'</span><span class="p">]</span><span class="o">.</span><span class="n">execution_date</span><span class="p">,</span> <span class="n">downstream_tasks</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Done."</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="PythonVirtualenvOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.python_operator.PythonVirtualenvOperator">[docs]</a><span class="k">class</span> <span class="nc">PythonVirtualenvOperator</span><span class="p">(</span><span class="n">PythonOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Allows one to run a function in a virtualenv that is created and destroyed</span>
+<span class="sd"> automatically (with certain caveats).</span>
+
+<span class="sd"> The function must be defined using def, and not be part of a class. All imports</span>
+<span class="sd"> must happen inside the function and no variables outside of the scope may be referenced.</span>
+<span class="sd"> A global scope variable named virtualenv_string_args will be available (populated by</span>
+<span class="sd"> string_args). In addition, one can pass stuff through op_args and op_kwargs, and one</span>
+<span class="sd"> can use a return value.</span>
+
+<span class="sd"> Note that if your virtualenv runs in a different Python major version than Airflow,</span>
+<span class="sd"> you cannot use return values, op_args, or op_kwargs. You can use string_args though.</span>
+
+<span class="sd"> :param python_callable: A python function with no references to outside variables,</span>
+<span class="sd"> defined with def, which will be run in a virtualenv</span>
+<span class="sd"> :type python_callable: function</span>
+<span class="sd"> :param requirements: A list of requirements as specified in a pip install command</span>
+<span class="sd"> :type requirements: list(str)</span>
+<span class="sd"> :param python_version: The Python version to run the virtualenv with. Note that</span>
+<span class="sd"> both 2 and 2.7 are acceptable forms.</span>
+<span class="sd"> :type python_version: str</span>
+<span class="sd"> :param use_dill: Whether to use dill to serialize the args and result (pickle is default).</span>
+<span class="sd"> This allow more complex types but requires you to include dill in your requirements.</span>
+<span class="sd"> :type use_dill: bool</span>
+<span class="sd"> :param system_site_packages: Whether to include system_site_packages in your virtualenv.</span>
+<span class="sd"> See virtualenv documentation for more information.</span>
+<span class="sd"> :type system_site_packages: bool</span>
+<span class="sd"> :param op_args: A list of positional arguments to pass to python_callable.</span>
+<span class="sd"> :type op_kwargs: list</span>
+<span class="sd"> :param op_kwargs: A dict of keyword arguments to pass to python_callable.</span>
+<span class="sd"> :type op_kwargs: dict</span>
+<span class="sd"> :param string_args: Strings that are present in the global var virtualenv_string_args,</span>
+<span class="sd"> available to python_callable at runtime as a list(str). Note that args are split</span>
+<span class="sd"> by newline.</span>
+<span class="sd"> :type string_args: list(str)</span>
+<span class="sd"> :param templates_dict: a dictionary where the values are templates that</span>
+<span class="sd"> will get templated by the Airflow engine sometime between</span>
+<span class="sd"> ``__init__`` and ``execute`` takes place and are made available</span>
+<span class="sd"> in your callable's context after the template has been applied</span>
+<span class="sd"> :type templates_dict: dict of str</span>
+<span class="sd"> :param templates_exts: a list of file extensions to resolve while</span>
+<span class="sd"> processing templated fields, for examples ``['.sql', '.hql']``</span>
+<span class="sd"> :type templates_exts: list(str)</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">python_callable</span><span class="p">,</span> <span class="n">requirements</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">python_version</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">use_dill</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">system_site_packages</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">op_args</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">op_kwargs</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">string_args</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">templates_dict</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">templates_exts</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">PythonVirtualenvOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+ <span class="n">python_callable</span><span class="o">=</span><span class="n">python_callable</span><span class="p">,</span>
+ <span class="n">op_args</span><span class="o">=</span><span class="n">op_args</span><span class="p">,</span>
+ <span class="n">op_kwargs</span><span class="o">=</span><span class="n">op_kwargs</span><span class="p">,</span>
+ <span class="n">templates_dict</span><span class="o">=</span><span class="n">templates_dict</span><span class="p">,</span>
+ <span class="n">templates_exts</span><span class="o">=</span><span class="n">templates_exts</span><span class="p">,</span>
+ <span class="n">provide_context</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">requirements</span> <span class="o">=</span> <span class="n">requirements</span> <span class="ow">or</span> <span class="p">[]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">string_args</span> <span class="o">=</span> <span class="n">string_args</span> <span class="ow">or</span> <span class="p">[]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">python_version</span> <span class="o">=</span> <span class="n">python_version</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">use_dill</span> <span class="o">=</span> <span class="n">use_dill</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">system_site_packages</span> <span class="o">=</span> <span class="n">system_site_packages</span>
+ <span class="c1"># check that dill is present if needed</span>
+ <span class="n">dill_in_requirements</span> <span class="o">=</span> <span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'dill'</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">requirements</span><span class="p">)</span>
+ <span class="k">if</span> <span class="p">(</span><span class="ow">not</span> <span class="n">system_site_packages</span><span class="p">)</span> <span class="ow">and</span> <span class="n">use_dill</span> <span class="ow">and</span> <span class="ow">not</span> <span class="nb">any</span><span class="p">(</span><span class="n">dill_in_requirements</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'If using dill, dill must be in the environment '</span> <span class="o">+</span>
+ <span class="s1">'either via system_site_packages or requirements'</span><span class="p">)</span>
+ <span class="c1"># check that a function is passed, and that it is not a lambda</span>
+ <span class="k">if</span> <span class="p">(</span><span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">python_callable</span><span class="p">,</span> <span class="n">types</span><span class="o">.</span><span class="n">FunctionType</span><span class="p">)</span>
+ <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">python_callable</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">==</span> <span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="mi">0</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'</span><span class="si">{}</span><span class="s1"> only supports functions for python_callable arg'</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="p">)</span>
+ <span class="c1"># check that args are passed iff python major version matches</span>
+ <span class="k">if</span> <span class="p">(</span><span class="n">python_version</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+ <span class="ow">and</span> <span class="nb">str</span><span class="p">(</span><span class="n">python_version</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> <span class="o">!=</span> <span class="nb">str</span><span class="p">(</span><span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
+ <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_pass_op_args</span><span class="p">()):</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Passing op_args or op_kwargs is not supported across "</span>
+ <span class="s2">"different Python major versions "</span>
+ <span class="s2">"for PythonVirtualenvOperator. Please use string_args."</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">execute_callable</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">with</span> <span class="n">TemporaryDirectory</span><span class="p">(</span><span class="n">prefix</span><span class="o">=</span><span class="s1">'venv'</span><span class="p">)</span> <span class="k">as</span> <span class="n">tmp_dir</span><span class="p">:</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">templates_dict</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">op_kwargs</span><span class="p">[</span><span class="s1">'templates_dict'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">templates_dict</span>
+ <span class="c1"># generate filenames</span>
+ <span class="n">input_filename</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">,</span> <span class="s1">'script.in'</span><span class="p">)</span>
+ <span class="n">output_filename</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">,</span> <span class="s1">'script.out'</span><span class="p">)</span>
+ <span class="n">string_args_filename</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">,</span> <span class="s1">'string_args.txt'</span><span class="p">)</span>
+ <span class="n">script_filename</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">,</span> <span class="s1">'script.py'</span><span class="p">)</span>
+
+ <span class="c1"># set up virtualenv</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_execute_in_subprocess</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_generate_virtualenv_cmd</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">))</span>
+ <span class="n">cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_generate_pip_install_cmd</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">cmd</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_execute_in_subprocess</span><span class="p">(</span><span class="n">cmd</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">_write_args</span><span class="p">(</span><span class="n">input_filename</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_write_script</span><span class="p">(</span><span class="n">script_filename</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_write_string_args</span><span class="p">(</span><span class="n">string_args_filename</span><span class="p">)</span>
+
+ <span class="c1"># execute command in virtualenv</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_execute_in_subprocess</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_generate_python_cmd</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">,</span>
+ <span class="n">script_filename</span><span class="p">,</span>
+ <span class="n">input_filename</span><span class="p">,</span>
+ <span class="n">output_filename</span><span class="p">,</span>
+ <span class="n">string_args_filename</span><span class="p">))</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_read_result</span><span class="p">(</span><span class="n">output_filename</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_pass_op_args</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="c1"># we should only pass op_args if any are given to us</span>
+ <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">op_args</span><span class="p">)</span> <span class="o">+</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">op_kwargs</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span>
+
+ <span class="k">def</span> <span class="nf">_execute_in_subprocess</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cmd</span><span class="p">):</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Executing cmd</span><span class="se">\n</span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">cmd</span><span class="p">))</span>
+ <span class="n">output</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">check_output</span><span class="p">(</span><span class="n">cmd</span><span class="p">,</span>
+ <span class="n">stderr</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">STDOUT</span><span class="p">,</span>
+ <span class="n">close_fds</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">output</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Got output</span><span class="se">\n</span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">output</span><span class="p">))</span>
+ <span class="k">except</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">CalledProcessError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Got error output</span><span class="se">\n</span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">output</span><span class="p">))</span>
+ <span class="k">raise</span>
+
+ <span class="k">def</span> <span class="nf">_write_string_args</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filename</span><span class="p">):</span>
+ <span class="c1"># writes string_args to a file, which are read line by line</span>
+ <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s1">'w'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
+ <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="nb">str</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">string_args</span><span class="p">)))</span>
+
+ <span class="k">def</span> <span class="nf">_write_args</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">input_filename</span><span class="p">):</span>
+ <span class="c1"># serialize args to file</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_pass_op_args</span><span class="p">():</span>
+ <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">input_filename</span><span class="p">,</span> <span class="s1">'wb'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
+ <span class="n">arg_dict</span> <span class="o">=</span> <span class="p">({</span><span class="s1">'args'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">op_args</span><span class="p">,</span> <span class="s1">'kwargs'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">op_kwargs</span><span class="p">})</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_dill</span><span class="p">:</span>
+ <span class="n">dill</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="n">arg_dict</span><span class="p">,</span> <span class="n">f</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">pickle</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="n">arg_dict</span><span class="p">,</span> <span class="n">f</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_read_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">output_filename</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">output_filename</span><span class="p">)</span><span class="o">.</span><span class="n">st_size</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">None</span>
+ <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">output_filename</span><span class="p">,</span> <span class="s1">'rb'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_dill</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">dill</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">pickle</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
+ <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Error deserializing result. Note that result deserialization "</span>
+ <span class="s2">"is not supported across major Python versions."</span><span class="p">)</span>
+ <span class="k">raise</span>
+
+ <span class="k">def</span> <span class="nf">_write_script</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">script_filename</span><span class="p">):</span>
+ <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">script_filename</span><span class="p">,</span> <span class="s1">'w'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
+ <span class="n">python_code</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_generate_python_code</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'Writing code to file</span><span class="se">\n</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">python_code</span><span class="p">))</span>
+ <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">python_code</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_generate_virtualenv_cmd</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tmp_dir</span><span class="p">):</span>
+ <span class="n">cmd</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'virtualenv'</span><span class="p">,</span> <span class="n">tmp_dir</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">system_site_packages</span><span class="p">:</span>
+ <span class="n">cmd</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s1">'--system-site-packages'</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">python_version</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">cmd</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s1">'--python=python</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">python_version</span><span class="p">))</span>
+ <span class="k">return</span> <span class="n">cmd</span>
+
+ <span class="k">def</span> <span class="nf">_generate_pip_install_cmd</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tmp_dir</span><span class="p">):</span>
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">requirements</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="k">return</span> <span class="p">[]</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="c1"># direct path alleviates need to activate</span>
+ <span class="n">cmd</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'</span><span class="si">{}</span><span class="s1">/bin/pip'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">),</span> <span class="s1">'install'</span><span class="p">]</span>
+ <span class="k">return</span> <span class="n">cmd</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">requirements</span>
+
+ <span class="k">def</span> <span class="nf">_generate_python_cmd</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tmp_dir</span><span class="p">,</span> <span class="n">script_filename</span><span class="p">,</span> <span class="n">input_filename</span><span class="p">,</span> <span class="n">output_filename</span><span class="p">,</span> <span class="n">string_args_filename</span><span class="p">):</span>
+ <span class="c1"># direct path alleviates need to activate</span>
+ <span class="k">return</span> <span class="p">[</span><span class="s1">'</span><span class="si">{}</span><span class="s1">/bin/python'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">),</span> <span class="n">script_filename</span><span class="p">,</span> <span class="n">input_filename</span><span class="p">,</span> <span class="n">output_filename</span><span class="p">,</span> <span class="n">string_args_filename</span><span class="p">]</span>
+
+ <span class="k">def</span> <span class="nf">_generate_python_code</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_dill</span><span class="p">:</span>
+ <span class="n">pickling_library</span> <span class="o">=</span> <span class="s1">'dill'</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">pickling_library</span> <span class="o">=</span> <span class="s1">'pickle'</span>
+ <span class="n">fn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">python_callable</span>
+ <span class="c1"># dont try to read pickle if we didnt pass anything</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_pass_op_args</span><span class="p">():</span>
+ <span class="n">load_args_line</span> <span class="o">=</span> <span class="s1">'with open(sys.argv[1], "rb") as f: arg_dict = </span><span class="si">{}</span><span class="s1">.load(f)'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">pickling_library</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">load_args_line</span> <span class="o">=</span> <span class="s1">'arg_dict = {"args": [], "kwargs": </span><span class="si">{}</span><span class="s1">}'</span>
+
+ <span class="c1"># no indents in original code so we can accept any type of indents in the original function</span>
+ <span class="c1"># we deserialize args, call function, serialize result if necessary</span>
+ <span class="k">return</span> <span class="n">dedent</span><span class="p">(</span><span class="s2">"""</span><span class="se">\</span>
+<span class="s2"> import </span><span class="si">{pickling_library}</span><span class="s2"></span>
+<span class="s2"> import sys</span>
+<span class="s2"> </span><span class="si">{load_args_code}</span><span class="s2"></span>
+<span class="s2"> args = arg_dict["args"]</span>
+<span class="s2"> kwargs = arg_dict["kwargs"]</span>
+<span class="s2"> with open(sys.argv[3], 'r') as f: virtualenv_string_args = list(map(lambda x: x.strip(), list(f)))</span>
+<span class="s2"> </span><span class="si">{python_callable_lines}</span><span class="s2"></span>
+<span class="s2"> res = </span><span class="si">{python_callable_name}</span><span class="s2">(*args, **kwargs)</span>
+<span class="s2"> with open(sys.argv[2], 'wb') as f: res is not None and </span><span class="si">{pickling_library}</span><span class="s2">.dump(res, f)</span>
+<span class="s2"> """</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">load_args_code</span><span class="o">=</span><span class="n">load_args_line</span><span class="p">,</span>
+ <span class="n">python_callable_lines</span><span class="o">=</span><span class="n">dedent</span><span class="p">(</span><span class="n">inspect</span><span class="o">.</span><span class="n">getsource</span><span class="p">(</span><span class="n">fn</span><span class="p">)),</span>
+ <span class="n">python_callable_name</span><span class="o">=</span><span class="n">fn</span><span class="o">.</span><span class="vm">__name__</span><span class="p">,</span>
+ <span class="n">pickling_library</span><span class="o">=</span><span class="n">pickling_library</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Done."</span><span class="p">)</span></div>
+
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/redshift_to_s3_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/redshift_to_s3_operator.html b/_modules/airflow/operators/redshift_to_s3_operator.html
index 6b8e248..a68f20e 100644
--- a/_modules/airflow/operators/redshift_to_s3_operator.html
+++ b/_modules/airflow/operators/redshift_to_s3_operator.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,22 @@
<h1>Source code for airflow.operators.redshift_to_s3_operator</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="kn">from</span> <span class="nn">airflow.hooks.postgres_hook</span> <span class="k">import</span> <span class="n">PostgresHook</span>
<span class="kn">from</span> <span class="nn">airflow.hooks.S3_hook</span> <span class="k">import</span> <span class="n">S3Hook</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
@@ -189,6 +196,7 @@
<div class="viewcode-block" id="RedshiftToS3Transfer"><a class="viewcode-back" href="../../../integration.html#airflow.operators.redshift_to_s3_operator.RedshiftToS3Transfer">[docs]</a><span class="k">class</span> <span class="nc">RedshiftToS3Transfer</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Executes an UNLOAD command to s3 as a CSV with headers</span>
+
<span class="sd"> :param schema: reference to a specific schema in redshift database</span>
<span class="sd"> :type schema: string</span>
<span class="sd"> :param table: reference to a specific table in redshift database</span>
@@ -201,8 +209,8 @@
<span class="sd"> :type redshift_conn_id: string</span>
<span class="sd"> :param aws_conn_id: reference to a specific S3 connection</span>
<span class="sd"> :type aws_conn_id: string</span>
-<span class="sd"> :param options: reference to a list of UNLOAD options</span>
-<span class="sd"> :type options: list</span>
+<span class="sd"> :param unload_options: reference to a list of UNLOAD options</span>
+<span class="sd"> :type unload_options: list</span>
<span class="sd"> """</span>
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">()</span>
@@ -221,6 +229,7 @@
<span class="n">unload_options</span><span class="o">=</span><span class="nb">tuple</span><span class="p">(),</span>
<span class="n">autocommit</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">include_header</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">RedshiftToS3Transfer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="o">=</span> <span class="n">schema</span>
@@ -232,41 +241,68 @@
<span class="bp">self</span><span class="o">.</span><span class="n">unload_options</span> <span class="o">=</span> <span class="n">unload_options</span>
<span class="bp">self</span><span class="o">.</span><span class="n">autocommit</span> <span class="o">=</span> <span class="n">autocommit</span>
<span class="bp">self</span><span class="o">.</span><span class="n">parameters</span> <span class="o">=</span> <span class="n">parameters</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">include_header</span> <span class="o">=</span> <span class="n">include_header</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">include_header</span> <span class="ow">and</span> \
+ <span class="s1">'PARALLEL OFF'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="n">uo</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="k">for</span> <span class="n">uo</span> <span class="ow">in</span> <span class="n">unload_options</span><span class="p">]:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">unload_options</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">unload_options</span><span class="p">)</span> <span class="o">+</span> <span class="p">[</span><span class="s1">'PARALLEL OFF'</span><span class="p">,</span> <span class="p">]</span>
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">PostgresHook</span><span class="p">(</span><span class="n">postgres_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">redshift_conn_id</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">s3</span> <span class="o">=</span> <span class="n">S3Hook</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span>
- <span class="n">a_key</span><span class="p">,</span> <span class="n">s_key</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">s3</span><span class="o">.</span><span class="n">get_credentials</span><span class="p">()</span>
+ <span class="n">credentials</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">s3</span><span class="o">.</span><span class="n">get_credentials</span><span class="p">()</span>
<span class="n">unload_options</span> <span class="o">=</span> <span class="s1">'</span><span class="se">\n\t\t\t</span><span class="s1">'</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">unload_options</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Retrieving headers from </span><span class="si">%s</span><span class="s2">.</span><span class="si">%s</span><span class="s2">..."</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="p">)</span>
-
- <span class="n">columns_query</span> <span class="o">=</span> <span class="s2">"""SELECT column_name</span>
-<span class="s2"> FROM information_schema.columns</span>
-<span class="s2"> WHERE table_schema = '</span><span class="si">{0}</span><span class="s2">'</span>
-<span class="s2"> AND table_name = '</span><span class="si">{1}</span><span class="s2">'</span>
-<span class="s2"> ORDER BY ordinal_position</span>
-<span class="s2"> """</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="p">)</span>
-
- <span class="n">cursor</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
- <span class="n">cursor</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">columns_query</span><span class="p">)</span>
- <span class="n">rows</span> <span class="o">=</span> <span class="n">cursor</span><span class="o">.</span><span class="n">fetchall</span><span class="p">()</span>
- <span class="n">columns</span> <span class="o">=</span> <span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">row</span><span class="p">:</span> <span class="n">row</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">rows</span><span class="p">)</span>
- <span class="n">column_names</span> <span class="o">=</span> <span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">c</span><span class="p">:</span> <span class="s2">"</span><span class="se">\\</span><span class="s2">'</span><span class="si">{0}</span><span class="se">\\</span><span class="s2">'"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">c</span><span class="p">),</span> <span class="n">columns</span><span class="p">))</span>
- <span class="n">column_castings</span> <span class="o">=</span> <span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">c</span><span class="p">:</span> <span class="s2">"CAST(</span><span class="si">{0}</span><span class="s2"> AS text) AS </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">c</span><span class="p">),</span>
- <span class="n">columns</span><span class="p">))</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">include_header</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Retrieving headers from </span><span class="si">%s</span><span class="s2">.</span><span class="si">%s</span><span class="s2">..."</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="p">)</span>
+
+ <span class="n">columns_query</span> <span class="o">=</span> <span class="s2">"""SELECT column_name</span>
+<span class="s2"> FROM information_schema.columns</span>
+<span class="s2"> WHERE table_schema = '</span><span class="si">{schema}</span><span class="s2">'</span>
+<span class="s2"> AND table_name = '</span><span class="si">{table}</span><span class="s2">'</span>
+<span class="s2"> ORDER BY ordinal_position</span>
+<span class="s2"> """</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">schema</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span>
+ <span class="n">table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="p">)</span>
+
+ <span class="n">cursor</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
+ <span class="n">cursor</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">columns_query</span><span class="p">)</span>
+ <span class="n">rows</span> <span class="o">=</span> <span class="n">cursor</span><span class="o">.</span><span class="n">fetchall</span><span class="p">()</span>
+ <span class="n">columns</span> <span class="o">=</span> <span class="p">[</span><span class="n">row</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">rows</span><span class="p">]</span>
+ <span class="n">column_names</span> <span class="o">=</span> <span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="s2">"</span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">c</span><span class="p">)</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">columns</span><span class="p">)</span>
+ <span class="n">column_headers</span> <span class="o">=</span> <span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="s2">"</span><span class="se">\\</span><span class="s2">'</span><span class="si">{0}</span><span class="se">\\</span><span class="s2">'"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">c</span><span class="p">)</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">columns</span><span class="p">)</span>
+ <span class="n">column_castings</span> <span class="o">=</span> <span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="s2">"CAST(</span><span class="si">{0}</span><span class="s2"> AS text) AS </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">c</span><span class="p">)</span>
+ <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">columns</span><span class="p">)</span>
+
+ <span class="n">select_query</span> <span class="o">=</span> <span class="s2">"""SELECT </span><span class="si">{column_names}</span><span class="s2"> FROM</span>
+<span class="s2"> (SELECT 2 sort_order, </span><span class="si">{column_castings}</span><span class="s2"></span>
+<span class="s2"> FROM </span><span class="si">{schema}</span><span class="s2">.</span><span class="si">{table}</span><span class="s2"></span>
+<span class="s2"> UNION ALL</span>
+<span class="s2"> SELECT 1 sort_order, </span><span class="si">{column_headers}</span><span class="s2">)</span>
+<span class="s2"> ORDER BY sort_order"""</span>\
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">column_names</span><span class="o">=</span><span class="n">column_names</span><span class="p">,</span>
+ <span class="n">column_castings</span><span class="o">=</span><span class="n">column_castings</span><span class="p">,</span>
+ <span class="n">column_headers</span><span class="o">=</span><span class="n">column_headers</span><span class="p">,</span>
+ <span class="n">schema</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span>
+ <span class="n">table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">select_query</span> <span class="o">=</span> <span class="s2">"SELECT * FROM </span><span class="si">{schema}</span><span class="s2">.</span><span class="si">{table}</span><span class="s2">"</span>\
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">schema</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span>
+ <span class="n">table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="p">)</span>
<span class="n">unload_query</span> <span class="o">=</span> <span class="s2">"""</span>
-<span class="s2"> UNLOAD ('SELECT </span><span class="si">{0}</span><span class="s2"></span>
-<span class="s2"> UNION ALL</span>
-<span class="s2"> SELECT </span><span class="si">{1}</span><span class="s2"> FROM </span><span class="si">{2}</span><span class="s2">.</span><span class="si">{3}</span><span class="s2"></span>
-<span class="s2"> ORDER BY 1 DESC')</span>
-<span class="s2"> TO 's3://</span><span class="si">{4}</span><span class="s2">/</span><span class="si">{5}</span><span class="s2">/</span><span class="si">{3}</span><span class="s2">_'</span>
-<span class="s2"> with</span>
-<span class="s2"> credentials 'aws_access_key_id=</span><span class="si">{6}</span><span class="s2">;aws_secret_access_key=</span><span class="si">{7}</span><span class="s2">'</span>
-<span class="s2"> </span><span class="si">{8}</span><span class="s2">;</span>
-<span class="s2"> """</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">column_names</span><span class="p">,</span> <span class="n">column_castings</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="p">,</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">s3_bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">s3_key</span><span class="p">,</span> <span class="n">a_key</span><span class="p">,</span> <span class="n">s_key</span><span class="p">,</span> <span class="n">unload_options</span><span class="p">)</span>
+<span class="s2"> UNLOAD ('</span><span class="si">{select_query}</span><span class="s2">')</span>
+<span class="s2"> TO 's3://</span><span class="si">{s3_bucket}</span><span class="s2">/</span><span class="si">{s3_key}</span><span class="s2">/</span><span class="si">{table}</span><span class="s2">_'</span>
+<span class="s2"> with credentials</span>
+<span class="s2"> 'aws_access_key_id=</span><span class="si">{access_key}</span><span class="s2">;aws_secret_access_key=</span><span class="si">{secret_key}</span><span class="s2">'</span>
+<span class="s2"> </span><span class="si">{unload_options}</span><span class="s2">;</span>
+<span class="s2"> """</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">select_query</span><span class="o">=</span><span class="n">select_query</span><span class="p">,</span>
+ <span class="n">table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="p">,</span>
+ <span class="n">s3_bucket</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_bucket</span><span class="p">,</span>
+ <span class="n">s3_key</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_key</span><span class="p">,</span>
+ <span class="n">access_key</span><span class="o">=</span><span class="n">credentials</span><span class="o">.</span><span class="n">access_key</span><span class="p">,</span>
+ <span class="n">secret_key</span><span class="o">=</span><span class="n">credentials</span><span class="o">.</span><span class="n">secret_key</span><span class="p">,</span>
+ <span class="n">unload_options</span><span class="o">=</span><span class="n">unload_options</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing UNLOAD command...'</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">unload_query</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">autocommit</span><span class="p">)</span>
[39/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/spark_submit_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/spark_submit_hook.html b/_modules/airflow/contrib/hooks/spark_submit_hook.html
new file mode 100644
index 0000000..6903f5f
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/spark_submit_hook.html
@@ -0,0 +1,799 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.spark_submit_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.spark_submit_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.spark_submit_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+<span class="kn">import</span> <span class="nn">os</span>
+<span class="kn">import</span> <span class="nn">subprocess</span>
+<span class="kn">import</span> <span class="nn">re</span>
+<span class="kn">import</span> <span class="nn">time</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.kubernetes</span> <span class="k">import</span> <span class="n">kube_client</span>
+<span class="kn">from</span> <span class="nn">kubernetes.client.rest</span> <span class="k">import</span> <span class="n">ApiException</span>
+
+
+<div class="viewcode-block" id="SparkSubmitHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.spark_submit_hook.SparkSubmitHook">[docs]</a><span class="k">class</span> <span class="nc">SparkSubmitHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> This hook is a wrapper around the spark-submit binary to kick off a spark-submit job.</span>
+<span class="sd"> It requires that the "spark-submit" binary is in the PATH or the spark_home to be</span>
+<span class="sd"> supplied.</span>
+<span class="sd"> :param conf: Arbitrary Spark configuration properties</span>
+<span class="sd"> :type conf: dict</span>
+<span class="sd"> :param conn_id: The connection id as configured in Airflow administration. When an</span>
+<span class="sd"> invalid connection_id is supplied, it will default to yarn.</span>
+<span class="sd"> :type conn_id: str</span>
+<span class="sd"> :param files: Upload additional files to the executor running the job, separated by a</span>
+<span class="sd"> comma. Files will be placed in the working directory of each executor.</span>
+<span class="sd"> For example, serialized objects.</span>
+<span class="sd"> :type files: str</span>
+<span class="sd"> :param py_files: Additional python files used by the job, can be .zip, .egg or .py.</span>
+<span class="sd"> :type py_files: str</span>
+<span class="sd"> :param driver_classpath: Additional, driver-specific, classpath settings.</span>
+<span class="sd"> :type driver_classpath: str</span>
+<span class="sd"> :param jars: Submit additional jars to upload and place them in executor classpath.</span>
+<span class="sd"> :type jars: str</span>
+<span class="sd"> :param java_class: the main class of the Java application</span>
+<span class="sd"> :type java_class: str</span>
+<span class="sd"> :param packages: Comma-separated list of maven coordinates of jars to include on the</span>
+<span class="sd"> driver and executor classpaths</span>
+<span class="sd"> :type packages: str</span>
+<span class="sd"> :param exclude_packages: Comma-separated list of maven coordinates of jars to exclude</span>
+<span class="sd"> while resolving the dependencies provided in 'packages'</span>
+<span class="sd"> :type exclude_packages: str</span>
+<span class="sd"> :param repositories: Comma-separated list of additional remote repositories to search</span>
+<span class="sd"> for the maven coordinates given with 'packages'</span>
+<span class="sd"> :type repositories: str</span>
+<span class="sd"> :param total_executor_cores: (Standalone & Mesos only) Total cores for all executors</span>
+<span class="sd"> (Default: all the available cores on the worker)</span>
+<span class="sd"> :type total_executor_cores: int</span>
+<span class="sd"> :param executor_cores: (Standalone, YARN and Kubernetes only) Number of cores per</span>
+<span class="sd"> executor (Default: 2)</span>
+<span class="sd"> :type executor_cores: int</span>
+<span class="sd"> :param executor_memory: Memory per executor (e.g. 1000M, 2G) (Default: 1G)</span>
+<span class="sd"> :type executor_memory: str</span>
+<span class="sd"> :param driver_memory: Memory allocated to the driver (e.g. 1000M, 2G) (Default: 1G)</span>
+<span class="sd"> :type driver_memory: str</span>
+<span class="sd"> :param keytab: Full path to the file that contains the keytab</span>
+<span class="sd"> :type keytab: str</span>
+<span class="sd"> :param principal: The name of the kerberos principal used for keytab</span>
+<span class="sd"> :type principal: str</span>
+<span class="sd"> :param name: Name of the job (default airflow-spark)</span>
+<span class="sd"> :type name: str</span>
+<span class="sd"> :param num_executors: Number of executors to launch</span>
+<span class="sd"> :type num_executors: int</span>
+<span class="sd"> :param application_args: Arguments for the application being submitted</span>
+<span class="sd"> :type application_args: list</span>
+<span class="sd"> :param env_vars: Environment variables for spark-submit. It</span>
+<span class="sd"> supports yarn and k8s mode too.</span>
+<span class="sd"> :type env_vars: dict</span>
+<span class="sd"> :param verbose: Whether to pass the verbose flag to spark-submit process for debugging</span>
+<span class="sd"> :type verbose: bool</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">conf</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">conn_id</span><span class="o">=</span><span class="s1">'spark_default'</span><span class="p">,</span>
+ <span class="n">files</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">py_files</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">driver_classpath</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">jars</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">java_class</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">packages</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">exclude_packages</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">repositories</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">total_executor_cores</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">executor_cores</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">executor_memory</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">driver_memory</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">keytab</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">principal</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">name</span><span class="o">=</span><span class="s1">'default-name'</span><span class="p">,</span>
+ <span class="n">num_executors</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">application_args</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">env_vars</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_conf</span> <span class="o">=</span> <span class="n">conf</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_conn_id</span> <span class="o">=</span> <span class="n">conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_files</span> <span class="o">=</span> <span class="n">files</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_py_files</span> <span class="o">=</span> <span class="n">py_files</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_driver_classpath</span> <span class="o">=</span> <span class="n">driver_classpath</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_jars</span> <span class="o">=</span> <span class="n">jars</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_java_class</span> <span class="o">=</span> <span class="n">java_class</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_packages</span> <span class="o">=</span> <span class="n">packages</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_exclude_packages</span> <span class="o">=</span> <span class="n">exclude_packages</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_repositories</span> <span class="o">=</span> <span class="n">repositories</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_total_executor_cores</span> <span class="o">=</span> <span class="n">total_executor_cores</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_executor_cores</span> <span class="o">=</span> <span class="n">executor_cores</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_executor_memory</span> <span class="o">=</span> <span class="n">executor_memory</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_driver_memory</span> <span class="o">=</span> <span class="n">driver_memory</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_keytab</span> <span class="o">=</span> <span class="n">keytab</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_principal</span> <span class="o">=</span> <span class="n">principal</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="n">name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_num_executors</span> <span class="o">=</span> <span class="n">num_executors</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_application_args</span> <span class="o">=</span> <span class="n">application_args</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_env_vars</span> <span class="o">=</span> <span class="n">env_vars</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_verbose</span> <span class="o">=</span> <span class="n">verbose</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_submit_sp</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_yarn_application_id</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_kubernetes_driver_pod</span> <span class="o">=</span> <span class="kc">None</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_resolve_connection</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_is_yarn</span> <span class="o">=</span> <span class="s1">'yarn'</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'master'</span><span class="p">]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_is_kubernetes</span> <span class="o">=</span> <span class="s1">'k8s'</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'master'</span><span class="p">]</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">_should_track_driver_status</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_resolve_should_track_driver_status</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_driver_id</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_driver_status</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_spark_exit_code</span> <span class="o">=</span> <span class="kc">None</span>
+
+ <span class="k">def</span> <span class="nf">_resolve_should_track_driver_status</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Determines whether or not this hook should poll the spark driver status through</span>
+<span class="sd"> subsequent spark-submit status requests after the initial spark-submit request</span>
+<span class="sd"> :return: if the driver status should be tracked</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="p">(</span><span class="s1">'spark://'</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'master'</span><span class="p">]</span> <span class="ow">and</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'deploy_mode'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'cluster'</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_resolve_connection</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="c1"># Build from connection master or default to yarn if not available</span>
+ <span class="n">conn_data</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'master'</span><span class="p">:</span> <span class="s1">'yarn'</span><span class="p">,</span>
+ <span class="s1">'queue'</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
+ <span class="s1">'deploy_mode'</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
+ <span class="s1">'spark_home'</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
+ <span class="s1">'spark_binary'</span><span class="p">:</span> <span class="s1">'spark-submit'</span><span class="p">,</span>
+ <span class="s1">'namespace'</span><span class="p">:</span> <span class="s1">'default'</span><span class="p">}</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="c1"># Master can be local, yarn, spark://HOST:PORT, mesos://HOST:PORT and</span>
+ <span class="c1"># k8s://https://<HOST>:<PORT></span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_conn_id</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">:</span>
+ <span class="n">conn_data</span><span class="p">[</span><span class="s1">'master'</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{}</span><span class="s2">:</span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">conn_data</span><span class="p">[</span><span class="s1">'master'</span><span class="p">]</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span>
+
+ <span class="c1"># Determine optional yarn queue from the extra field</span>
+ <span class="n">extra</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span>
+ <span class="n">conn_data</span><span class="p">[</span><span class="s1">'queue'</span><span class="p">]</span> <span class="o">=</span> <span class="n">extra</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'queue'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="n">conn_data</span><span class="p">[</span><span class="s1">'deploy_mode'</span><span class="p">]</span> <span class="o">=</span> <span class="n">extra</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'deploy-mode'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="n">conn_data</span><span class="p">[</span><span class="s1">'spark_home'</span><span class="p">]</span> <span class="o">=</span> <span class="n">extra</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'spark-home'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="n">conn_data</span><span class="p">[</span><span class="s1">'spark_binary'</span><span class="p">]</span> <span class="o">=</span> <span class="n">extra</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'spark-binary'</span><span class="p">,</span> <span class="s1">'spark-submit'</span><span class="p">)</span>
+ <span class="n">conn_data</span><span class="p">[</span><span class="s1">'namespace'</span><span class="p">]</span> <span class="o">=</span> <span class="n">extra</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'namespace'</span><span class="p">,</span> <span class="s1">'default'</span><span class="p">)</span>
+ <span class="k">except</span> <span class="n">AirflowException</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
+ <span class="s2">"Could not load connection string </span><span class="si">%s</span><span class="s2">, defaulting to </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_conn_id</span><span class="p">,</span> <span class="n">conn_data</span><span class="p">[</span><span class="s1">'master'</span><span class="p">]</span>
+ <span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">conn_data</span>
+
+ <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">pass</span>
+
+ <span class="k">def</span> <span class="nf">_get_spark_binary_path</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="c1"># If the spark_home is passed then build the spark-submit executable path using</span>
+ <span class="c1"># the spark_home; otherwise assume that spark-submit is present in the path to</span>
+ <span class="c1"># the executing user</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'spark_home'</span><span class="p">]:</span>
+ <span class="n">connection_cmd</span> <span class="o">=</span> <span class="p">[</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'spark_home'</span><span class="p">],</span> <span class="s1">'bin'</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'spark_binary'</span><span class="p">])]</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'spark_binary'</span><span class="p">]]</span>
+
+ <span class="k">return</span> <span class="n">connection_cmd</span>
+
+ <span class="k">def</span> <span class="nf">_build_spark_submit_command</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">application</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Construct the spark-submit command to execute.</span>
+<span class="sd"> :param application: command to append to the spark-submit command</span>
+<span class="sd"> :type application: str</span>
+<span class="sd"> :return: full command to be executed</span>
+<span class="sd"> """</span>
+ <span class="n">connection_cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_spark_binary_path</span><span class="p">()</span>
+
+ <span class="c1"># The url ot the spark master</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--master"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'master'</span><span class="p">]]</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_conf</span><span class="p">:</span>
+ <span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_conf</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--conf"</span><span class="p">,</span> <span class="s2">"</span><span class="si">{}</span><span class="s2">=</span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_conf</span><span class="p">[</span><span class="n">key</span><span class="p">]))]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_env_vars</span> <span class="ow">and</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_is_kubernetes</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">_is_yarn</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_is_yarn</span><span class="p">:</span>
+ <span class="n">tmpl</span> <span class="o">=</span> <span class="s2">"spark.yarn.appMasterEnv.</span><span class="si">{}</span><span class="s2">=</span><span class="si">{}</span><span class="s2">"</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">tmpl</span> <span class="o">=</span> <span class="s2">"spark.kubernetes.driverEnv.</span><span class="si">{}</span><span class="s2">=</span><span class="si">{}</span><span class="s2">"</span>
+ <span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_env_vars</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span>
+ <span class="s2">"--conf"</span><span class="p">,</span>
+ <span class="n">tmpl</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_env_vars</span><span class="p">[</span><span class="n">key</span><span class="p">]))]</span>
+ <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_env_vars</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'deploy_mode'</span><span class="p">]</span> <span class="o">!=</span> <span class="s2">"cluster"</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_env</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_env_vars</span> <span class="c1"># Do it on Popen of the process</span>
+ <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_env_vars</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'deploy_mode'</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"cluster"</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s2">"SparkSubmitHook env_vars is not supported in standalone-cluster mode."</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_is_kubernetes</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--conf"</span><span class="p">,</span> <span class="s2">"spark.kubernetes.namespace=</span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'namespace'</span><span class="p">])]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_files</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--files"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_files</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_py_files</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--py-files"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_py_files</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_driver_classpath</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--driver-classpath"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_driver_classpath</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jars</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--jars"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jars</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_packages</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--packages"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_packages</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_exclude_packages</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--exclude-packages"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_exclude_packages</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_repositories</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--repositories"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_repositories</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_num_executors</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--num-executors"</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_num_executors</span><span class="p">)]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_total_executor_cores</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--total-executor-cores"</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_total_executor_cores</span><span class="p">)]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_executor_cores</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--executor-cores"</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_executor_cores</span><span class="p">)]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_executor_memory</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--executor-memory"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_executor_memory</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_driver_memory</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--driver-memory"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_driver_memory</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_keytab</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--keytab"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_keytab</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_principal</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--principal"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_principal</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_name</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--name"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_name</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_java_class</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--class"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_java_class</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_verbose</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--verbose"</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'queue'</span><span class="p">]:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--queue"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'queue'</span><span class="p">]]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'deploy_mode'</span><span class="p">]:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--deploy-mode"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'deploy_mode'</span><span class="p">]]</span>
+
+ <span class="c1"># The actual script to execute</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="n">application</span><span class="p">]</span>
+
+ <span class="c1"># Append any application arguments</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_application_args</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_application_args</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Spark-Submit cmd: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">connection_cmd</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">connection_cmd</span>
+
+ <span class="k">def</span> <span class="nf">_build_track_driver_status_command</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Construct the command to poll the driver status.</span>
+
+<span class="sd"> :return: full command to be executed</span>
+<span class="sd"> """</span>
+ <span class="n">connection_cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_spark_binary_path</span><span class="p">()</span>
+
+ <span class="c1"># The url ot the spark master</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--master"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'master'</span><span class="p">]]</span>
+
+ <span class="c1"># The driver id so we can poll for its status</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_driver_id</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--status"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_driver_id</span><span class="p">]</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s2">"Invalid status: attempted to poll driver "</span> <span class="o">+</span>
+ <span class="s2">"status but no driver id is known. Giving up."</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Poll driver status cmd: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">connection_cmd</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">connection_cmd</span>
+
+<div class="viewcode-block" id="SparkSubmitHook.submit"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.spark_submit_hook.SparkSubmitHook.submit">[docs]</a> <span class="k">def</span> <span class="nf">submit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">application</span><span class="o">=</span><span class="s2">""</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Remote Popen to execute the spark-submit job</span>
+
+<span class="sd"> :param application: Submitted application, jar or py file</span>
+<span class="sd"> :type application: str</span>
+<span class="sd"> :param kwargs: extra arguments to Popen (see subprocess.Popen)</span>
+<span class="sd"> """</span>
+ <span class="n">spark_submit_cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_spark_submit_command</span><span class="p">(</span><span class="n">application</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">'_env'</span><span class="p">):</span>
+ <span class="n">env</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+ <span class="n">env</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_env</span><span class="p">)</span>
+ <span class="n">kwargs</span><span class="p">[</span><span class="s2">"env"</span><span class="p">]</span> <span class="o">=</span> <span class="n">env</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">_submit_sp</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">Popen</span><span class="p">(</span><span class="n">spark_submit_cmd</span><span class="p">,</span>
+ <span class="n">stdout</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span>
+ <span class="n">stderr</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">STDOUT</span><span class="p">,</span>
+ <span class="n">bufsize</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span>
+ <span class="n">universal_newlines</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">_process_spark_submit_log</span><span class="p">(</span><span class="nb">iter</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_submit_sp</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">readline</span><span class="p">,</span> <span class="s1">''</span><span class="p">))</span>
+ <span class="n">returncode</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_submit_sp</span><span class="o">.</span><span class="n">wait</span><span class="p">()</span>
+
+ <span class="c1"># Check spark-submit return code. In Kubernetes mode, also check the value</span>
+ <span class="c1"># of exit code in the log, as it may differ.</span>
+ <span class="k">if</span> <span class="n">returncode</span> <span class="ow">or</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_is_kubernetes</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_spark_exit_code</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s2">"Cannot execute: </span><span class="si">{}</span><span class="s2">. Error code is: </span><span class="si">{}</span><span class="s2">."</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">spark_submit_cmd</span><span class="p">,</span> <span class="n">returncode</span>
+ <span class="p">)</span>
+ <span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Should track driver: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_should_track_driver_status</span><span class="p">))</span>
+
+ <span class="c1"># We want the Airflow job to wait until the Spark driver is finished</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_should_track_driver_status</span><span class="p">:</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_driver_id</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s2">"No driver id is known: something went wrong when executing "</span> <span class="o">+</span>
+ <span class="s2">"the spark submit command"</span>
+ <span class="p">)</span>
+
+ <span class="c1"># We start with the SUBMITTED status as initial status</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_driver_status</span> <span class="o">=</span> <span class="s2">"SUBMITTED"</span>
+
+ <span class="c1"># Start tracking the driver status (blocking function)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_start_driver_status_tracking</span><span class="p">()</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_driver_status</span> <span class="o">!=</span> <span class="s2">"FINISHED"</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s2">"ERROR : Driver </span><span class="si">{}</span><span class="s2"> badly exited with status </span><span class="si">{}</span><span class="s2">"</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_driver_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_driver_status</span><span class="p">)</span>
+ <span class="p">)</span></div>
+
+ <span class="k">def</span> <span class="nf">_process_spark_submit_log</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">itr</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Processes the log files and extracts useful information out of it.</span>
+
+<span class="sd"> If the deploy-mode is 'client', log the output of the submit command as those</span>
+<span class="sd"> are the output logs of the Spark worker directly.</span>
+
+<span class="sd"> Remark: If the driver needs to be tracked for its status, the log-level of the</span>
+<span class="sd"> spark deploy needs to be at least INFO (log4j.logger.org.apache.spark.deploy=INFO)</span>
+
+<span class="sd"> :param itr: An iterator which iterates over the input of the subprocess</span>
+<span class="sd"> """</span>
+ <span class="c1"># Consume the iterator</span>
+ <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">itr</span><span class="p">:</span>
+ <span class="n">line</span> <span class="o">=</span> <span class="n">line</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
+ <span class="c1"># If we run yarn cluster mode, we want to extract the application id from</span>
+ <span class="c1"># the logs so we can kill the application when we stop it unexpectedly</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_is_yarn</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'deploy_mode'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'cluster'</span><span class="p">:</span>
+ <span class="n">match</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="s1">'(application[0-9_]+)'</span><span class="p">,</span> <span class="n">line</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">match</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_yarn_application_id</span> <span class="o">=</span> <span class="n">match</span><span class="o">.</span><span class="n">groups</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Identified spark driver id: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_yarn_application_id</span><span class="p">)</span>
+
+ <span class="c1"># If we run Kubernetes cluster mode, we want to extract the driver pod id</span>
+ <span class="c1"># from the logs so we can kill the application when we stop it unexpectedly</span>
+ <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_is_kubernetes</span><span class="p">:</span>
+ <span class="n">match</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="s1">'\s*pod name: ((.+?)-([a-z0-9]+)-driver)'</span><span class="p">,</span> <span class="n">line</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">match</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_kubernetes_driver_pod</span> <span class="o">=</span> <span class="n">match</span><span class="o">.</span><span class="n">groups</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Identified spark driver pod: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_kubernetes_driver_pod</span><span class="p">)</span>
+
+ <span class="c1"># Store the Spark Exit code</span>
+ <span class="n">match_exit_code</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="s1">'\s*Exit code: (\d+)'</span><span class="p">,</span> <span class="n">line</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">match_exit_code</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_spark_exit_code</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">match_exit_code</span><span class="o">.</span><span class="n">groups</span><span class="p">()[</span><span class="mi">0</span><span class="p">])</span>
+
+ <span class="c1"># if we run in standalone cluster mode and we want to track the driver status</span>
+ <span class="c1"># we need to extract the driver id from the logs. This allows us to poll for</span>
+ <span class="c1"># the status using the driver id. Also, we can kill the driver when needed.</span>
+ <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_should_track_driver_status</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_driver_id</span><span class="p">:</span>
+ <span class="n">match_driver_id</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="s1">'(driver-[0-9\-]+)'</span><span class="p">,</span> <span class="n">line</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">match_driver_id</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_driver_id</span> <span class="o">=</span> <span class="n">match_driver_id</span><span class="o">.</span><span class="n">groups</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"identified spark driver id: </span><span class="si">{}</span><span class="s2">"</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_driver_id</span><span class="p">))</span>
+
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">line</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"spark submit log: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">line</span><span class="p">))</span>
+
+ <span class="k">def</span> <span class="nf">_process_spark_status_log</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">itr</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> parses the logs of the spark driver status query process</span>
+
+<span class="sd"> :param itr: An iterator which iterates over the input of the subprocess</span>
+<span class="sd"> """</span>
+ <span class="c1"># Consume the iterator</span>
+ <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">itr</span><span class="p">:</span>
+ <span class="n">line</span> <span class="o">=</span> <span class="n">line</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
+
+ <span class="c1"># Check if the log line is about the driver status and extract the status.</span>
+ <span class="k">if</span> <span class="s2">"driverState"</span> <span class="ow">in</span> <span class="n">line</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_driver_status</span> <span class="o">=</span> <span class="n">line</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">' : '</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span> \
+ <span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">','</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="se">\"</span><span class="s1">'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"spark driver status log: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">line</span><span class="p">))</span>
+
+ <span class="k">def</span> <span class="nf">_start_driver_status_tracking</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Polls the driver based on self._driver_id to get the status.</span>
+<span class="sd"> Finish successfully when the status is FINISHED.</span>
+<span class="sd"> Finish failed when the status is ERROR/UNKNOWN/KILLED/FAILED.</span>
+
+<span class="sd"> Possible status:</span>
+<span class="sd"> SUBMITTED: Submitted but not yet scheduled on a worker</span>
+<span class="sd"> RUNNING: Has been allocated to a worker to run</span>
+<span class="sd"> FINISHED: Previously ran and exited cleanly</span>
+<span class="sd"> RELAUNCHING: Exited non-zero or due to worker failure, but has not yet</span>
+<span class="sd"> started running again</span>
+<span class="sd"> UNKNOWN: The status of the driver is temporarily not known due to</span>
+<span class="sd"> master failure recovery</span>
+<span class="sd"> KILLED: A user manually killed this driver</span>
+<span class="sd"> FAILED: The driver exited non-zero and was not supervised</span>
+<span class="sd"> ERROR: Unable to run or restart due to an unrecoverable error</span>
+<span class="sd"> (e.g. missing jar file)</span>
+<span class="sd"> """</span>
+
+ <span class="c1"># When your Spark Standalone cluster is not performing well</span>
+ <span class="c1"># due to misconfiguration or heavy loads.</span>
+ <span class="c1"># it is possible that the polling request will timeout.</span>
+ <span class="c1"># Therefore we use a simple retry mechanism.</span>
+ <span class="n">missed_job_status_reports</span> <span class="o">=</span> <span class="mi">0</span>
+ <span class="n">max_missed_job_status_reports</span> <span class="o">=</span> <span class="mi">10</span>
+
+ <span class="c1"># Keep polling as long as the driver is processing</span>
+ <span class="k">while</span> <span class="bp">self</span><span class="o">.</span><span class="n">_driver_status</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">"FINISHED"</span><span class="p">,</span> <span class="s2">"UNKNOWN"</span><span class="p">,</span>
+ <span class="s2">"KILLED"</span><span class="p">,</span> <span class="s2">"FAILED"</span><span class="p">,</span> <span class="s2">"ERROR"</span><span class="p">]:</span>
+
+ <span class="c1"># Sleep for 1 second as we do not want to spam the cluster</span>
+ <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"polling status of spark driver with id </span><span class="si">{}</span><span class="s2">"</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_driver_id</span><span class="p">))</span>
+
+ <span class="n">poll_drive_status_cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_track_driver_status_command</span><span class="p">()</span>
+ <span class="n">status_process</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">Popen</span><span class="p">(</span><span class="n">poll_drive_status_cmd</span><span class="p">,</span>
+ <span class="n">stdout</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span>
+ <span class="n">stderr</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">STDOUT</span><span class="p">,</span>
+ <span class="n">bufsize</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span>
+ <span class="n">universal_newlines</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">_process_spark_status_log</span><span class="p">(</span><span class="nb">iter</span><span class="p">(</span><span class="n">status_process</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">readline</span><span class="p">,</span> <span class="s1">''</span><span class="p">))</span>
+ <span class="n">returncode</span> <span class="o">=</span> <span class="n">status_process</span><span class="o">.</span><span class="n">wait</span><span class="p">()</span>
+
+ <span class="k">if</span> <span class="n">returncode</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">missed_job_status_reports</span> <span class="o"><</span> <span class="n">max_missed_job_status_reports</span><span class="p">:</span>
+ <span class="n">missed_job_status_reports</span> <span class="o">=</span> <span class="n">missed_job_status_reports</span> <span class="o">+</span> <span class="mi">1</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s2">"Failed to poll for the driver status </span><span class="si">{}</span><span class="s2"> times: returncode = </span><span class="si">{}</span><span class="s2">"</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">max_missed_job_status_reports</span><span class="p">,</span> <span class="n">returncode</span><span class="p">)</span>
+ <span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_build_spark_driver_kill_command</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Construct the spark-submit command to kill a driver.</span>
+<span class="sd"> :return: full command to kill a driver</span>
+<span class="sd"> """</span>
+
+ <span class="c1"># If the spark_home is passed then build the spark-submit executable path using</span>
+ <span class="c1"># the spark_home; otherwise assume that spark-submit is present in the path to</span>
+ <span class="c1"># the executing user</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'spark_home'</span><span class="p">]:</span>
+ <span class="n">connection_cmd</span> <span class="o">=</span> <span class="p">[</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'spark_home'</span><span class="p">],</span>
+ <span class="s1">'bin'</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'spark_binary'</span><span class="p">])]</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">connection_cmd</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'spark_binary'</span><span class="p">]]</span>
+
+ <span class="c1"># The url ot the spark master</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--master"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'master'</span><span class="p">]]</span>
+
+ <span class="c1"># The actual kill command</span>
+ <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--kill"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_driver_id</span><span class="p">]</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Spark-Kill cmd: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">connection_cmd</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">connection_cmd</span>
+
+ <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Kill Command is being called"</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_should_track_driver_status</span><span class="p">:</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_driver_id</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Killing driver </span><span class="si">{}</span><span class="s1"> on cluster'</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_driver_id</span><span class="p">))</span>
+
+ <span class="n">kill_cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_spark_driver_kill_command</span><span class="p">()</span>
+ <span class="n">driver_kill</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">Popen</span><span class="p">(</span><span class="n">kill_cmd</span><span class="p">,</span>
+ <span class="n">stdout</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span>
+ <span class="n">stderr</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Spark driver </span><span class="si">{}</span><span class="s2"> killed with return code: </span><span class="si">{}</span><span class="s2">"</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_driver_id</span><span class="p">,</span> <span class="n">driver_kill</span><span class="o">.</span><span class="n">wait</span><span class="p">()))</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_submit_sp</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_submit_sp</span><span class="o">.</span><span class="n">poll</span><span class="p">()</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Sending kill signal to </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'spark_binary'</span><span class="p">])</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_submit_sp</span><span class="o">.</span><span class="n">kill</span><span class="p">()</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_yarn_application_id</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Killing application </span><span class="si">{}</span><span class="s1"> on YARN'</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_yarn_application_id</span><span class="p">))</span>
+
+ <span class="n">kill_cmd</span> <span class="o">=</span> <span class="s2">"yarn application -kill </span><span class="si">{}</span><span class="s2">"</span> \
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_yarn_application_id</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">()</span>
+ <span class="n">yarn_kill</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">Popen</span><span class="p">(</span><span class="n">kill_cmd</span><span class="p">,</span>
+ <span class="n">stdout</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span>
+ <span class="n">stderr</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"YARN killed with return code: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">yarn_kill</span><span class="o">.</span><span class="n">wait</span><span class="p">())</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_kubernetes_driver_pod</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Killing pod </span><span class="si">%s</span><span class="s1"> on Kubernetes'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_kubernetes_driver_pod</span><span class="p">)</span>
+
+ <span class="c1"># Currently only instantiate Kubernetes client for killing a spark pod.</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">client</span> <span class="o">=</span> <span class="n">kube_client</span><span class="o">.</span><span class="n">get_kube_client</span><span class="p">()</span>
+ <span class="n">api_response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">delete_namespaced_pod</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_kubernetes_driver_pod</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'namespace'</span><span class="p">],</span>
+ <span class="n">body</span><span class="o">=</span><span class="n">client</span><span class="o">.</span><span class="n">V1DeleteOptions</span><span class="p">(),</span>
+ <span class="n">pretty</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Spark on K8s killed with response: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">api_response</span><span class="p">)</span>
+
+ <span class="k">except</span> <span class="n">ApiException</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Exception when attempting to kill Spark on K8s:"</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="n">e</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[37/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/ssh_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/ssh_hook.html b/_modules/airflow/contrib/hooks/ssh_hook.html
new file mode 100644
index 0000000..08f1831
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/ssh_hook.html
@@ -0,0 +1,470 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.ssh_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.ssh_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.ssh_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Copyright 2012-2015 Spotify AB</span>
+<span class="c1"># Ported to Airflow by Bolke de Bruin</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">getpass</span>
+<span class="kn">import</span> <span class="nn">os</span>
+
+<span class="kn">import</span> <span class="nn">paramiko</span>
+<span class="kn">from</span> <span class="nn">paramiko.config</span> <span class="k">import</span> <span class="n">SSH_PORT</span>
+
+<span class="kn">from</span> <span class="nn">contextlib</span> <span class="k">import</span> <span class="n">contextmanager</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+
+<div class="viewcode-block" id="SSHHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ssh_hook.SSHHook">[docs]</a><span class="k">class</span> <span class="nc">SSHHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Hook for ssh remote execution using Paramiko.</span>
+<span class="sd"> ref: https://github.com/paramiko/paramiko</span>
+<span class="sd"> This hook also lets you create ssh tunnel and serve as basis for SFTP file transfer</span>
+
+<span class="sd"> :param ssh_conn_id: connection id from airflow Connections from where all the required</span>
+<span class="sd"> parameters can be fetched like username, password or key_file.</span>
+<span class="sd"> Thought the priority is given to the param passed during init</span>
+<span class="sd"> :type ssh_conn_id: str</span>
+<span class="sd"> :param remote_host: remote host to connect</span>
+<span class="sd"> :type remote_host: str</span>
+<span class="sd"> :param username: username to connect to the remote_host</span>
+<span class="sd"> :type username: str</span>
+<span class="sd"> :param password: password of the username to connect to the remote_host</span>
+<span class="sd"> :type password: str</span>
+<span class="sd"> :param key_file: key file to use to connect to the remote_host.</span>
+<span class="sd"> :type key_file: str</span>
+<span class="sd"> :param port: port of remote host to connect (Default is paramiko SSH_PORT)</span>
+<span class="sd"> :type port: int</span>
+<span class="sd"> :param timeout: timeout for the attempt to connect to the remote_host.</span>
+<span class="sd"> :type timeout: int</span>
+<span class="sd"> :param keepalive_interval: send a keepalive packet to remote host every keepalive_interval seconds</span>
+<span class="sd"> :type keepalive_interval: int</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">ssh_conn_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">remote_host</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">username</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">password</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">key_file</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">port</span><span class="o">=</span><span class="n">SSH_PORT</span><span class="p">,</span>
+ <span class="n">timeout</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span>
+ <span class="n">keepalive_interval</span><span class="o">=</span><span class="mi">30</span>
+ <span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">SSHHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">ssh_conn_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span> <span class="o">=</span> <span class="n">ssh_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span> <span class="o">=</span> <span class="n">remote_host</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="o">=</span> <span class="n">username</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="o">=</span> <span class="n">password</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span> <span class="o">=</span> <span class="n">key_file</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span> <span class="o">=</span> <span class="n">timeout</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">keepalive_interval</span> <span class="o">=</span> <span class="n">keepalive_interval</span>
+ <span class="c1"># Default values, overridable from Connection</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">compress</span> <span class="o">=</span> <span class="kc">True</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">no_host_key_check</span> <span class="o">=</span> <span class="kc">True</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">port</span> <span class="o">=</span> <span class="n">port</span>
+
+ <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'Creating SSH client for conn_id: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span>
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">port</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span>
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">extra_options</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span> <span class="o">=</span> <span class="n">extra_options</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"key_file"</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="s2">"timeout"</span> <span class="ow">in</span> <span class="n">extra_options</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">extra_options</span><span class="p">[</span><span class="s2">"timeout"</span><span class="p">],</span> <span class="mi">10</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="s2">"compress"</span> <span class="ow">in</span> <span class="n">extra_options</span> \
+ <span class="ow">and</span> <span class="nb">str</span><span class="p">(</span><span class="n">extra_options</span><span class="p">[</span><span class="s2">"compress"</span><span class="p">])</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'false'</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">compress</span> <span class="o">=</span> <span class="kc">False</span>
+ <span class="k">if</span> <span class="s2">"no_host_key_check"</span> <span class="ow">in</span> <span class="n">extra_options</span> \
+ <span class="ow">and</span> \
+ <span class="nb">str</span><span class="p">(</span><span class="n">extra_options</span><span class="p">[</span><span class="s2">"no_host_key_check"</span><span class="p">])</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'false'</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">no_host_key_check</span> <span class="o">=</span> <span class="kc">False</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Missing required param: remote_host"</span><span class="p">)</span>
+
+ <span class="c1"># Auto detecting username values from system</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
+ <span class="s2">"username to ssh to host: </span><span class="si">%s</span><span class="s2"> is not specified for connection id"</span>
+ <span class="s2">" </span><span class="si">%s</span><span class="s2">. Using system's default provided by getpass.getuser()"</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span>
+ <span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="o">=</span> <span class="n">getpass</span><span class="o">.</span><span class="n">getuser</span><span class="p">()</span>
+
+ <span class="n">host_proxy</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="n">user_ssh_config_filename</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">expanduser</span><span class="p">(</span><span class="s1">'~/.ssh/config'</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">user_ssh_config_filename</span><span class="p">):</span>
+ <span class="n">ssh_conf</span> <span class="o">=</span> <span class="n">paramiko</span><span class="o">.</span><span class="n">SSHConfig</span><span class="p">()</span>
+ <span class="n">ssh_conf</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="nb">open</span><span class="p">(</span><span class="n">user_ssh_config_filename</span><span class="p">))</span>
+ <span class="n">host_info</span> <span class="o">=</span> <span class="n">ssh_conf</span><span class="o">.</span><span class="n">lookup</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">host_info</span> <span class="ow">and</span> <span class="n">host_info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'proxycommand'</span><span class="p">):</span>
+ <span class="n">host_proxy</span> <span class="o">=</span> <span class="n">paramiko</span><span class="o">.</span><span class="n">ProxyCommand</span><span class="p">(</span><span class="n">host_info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'proxycommand'</span><span class="p">))</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">host_info</span> <span class="ow">and</span> <span class="n">host_info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'identityfile'</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span> <span class="o">=</span> <span class="n">host_info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'identityfile'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">client</span> <span class="o">=</span> <span class="n">paramiko</span><span class="o">.</span><span class="n">SSHClient</span><span class="p">()</span>
+ <span class="n">client</span><span class="o">.</span><span class="n">load_system_host_keys</span><span class="p">()</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">no_host_key_check</span><span class="p">:</span>
+ <span class="c1"># Default is RejectPolicy</span>
+ <span class="n">client</span><span class="o">.</span><span class="n">set_missing_host_key_policy</span><span class="p">(</span><span class="n">paramiko</span><span class="o">.</span><span class="n">AutoAddPolicy</span><span class="p">())</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span><span class="o">.</span><span class="n">strip</span><span class="p">():</span>
+ <span class="n">client</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">hostname</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span>
+ <span class="n">username</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">,</span>
+ <span class="n">password</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">password</span><span class="p">,</span>
+ <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">,</span>
+ <span class="n">compress</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">compress</span><span class="p">,</span>
+ <span class="n">port</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">port</span><span class="p">,</span>
+ <span class="n">sock</span><span class="o">=</span><span class="n">host_proxy</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">client</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">hostname</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span>
+ <span class="n">username</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">,</span>
+ <span class="n">key_filename</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">key_file</span><span class="p">,</span>
+ <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">,</span>
+ <span class="n">compress</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">compress</span><span class="p">,</span>
+ <span class="n">port</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">port</span><span class="p">,</span>
+ <span class="n">sock</span><span class="o">=</span><span class="n">host_proxy</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">keepalive_interval</span><span class="p">:</span>
+ <span class="n">client</span><span class="o">.</span><span class="n">get_transport</span><span class="p">()</span><span class="o">.</span><span class="n">set_keepalive</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">keepalive_interval</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="n">client</span>
+ <span class="k">except</span> <span class="n">paramiko</span><span class="o">.</span><span class="n">AuthenticationException</span> <span class="k">as</span> <span class="n">auth_error</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
+ <span class="s2">"Auth failed while connecting to host: </span><span class="si">%s</span><span class="s2">, error: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span> <span class="n">auth_error</span>
+ <span class="p">)</span>
+ <span class="k">except</span> <span class="n">paramiko</span><span class="o">.</span><span class="n">SSHException</span> <span class="k">as</span> <span class="n">ssh_error</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
+ <span class="s2">"Failed connecting to host: </span><span class="si">%s</span><span class="s2">, error: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span> <span class="n">ssh_error</span>
+ <span class="p">)</span>
+ <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
+ <span class="s2">"Error connecting to host: </span><span class="si">%s</span><span class="s2">, error: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span> <span class="n">error</span>
+ <span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span>
+
+<div class="viewcode-block" id="SSHHook.create_tunnel"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ssh_hook.SSHHook.create_tunnel">[docs]</a> <span class="nd">@contextmanager</span>
+ <span class="k">def</span> <span class="nf">create_tunnel</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">local_port</span><span class="p">,</span> <span class="n">remote_port</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">remote_host</span><span class="o">=</span><span class="s2">"localhost"</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Creates a tunnel between two hosts. Like ssh -L <LOCAL_PORT>:host:<REMOTE_PORT>.</span>
+<span class="sd"> Remember to close() the returned "tunnel" object in order to clean up</span>
+<span class="sd"> after yourself when you are done with the tunnel.</span>
+
+<span class="sd"> :param local_port:</span>
+<span class="sd"> :type local_port: int</span>
+<span class="sd"> :param remote_port:</span>
+<span class="sd"> :type remote_port: int</span>
+<span class="sd"> :param remote_host:</span>
+<span class="sd"> :type remote_host: str</span>
+<span class="sd"> :return:</span>
+<span class="sd"> """</span>
+
+ <span class="kn">import</span> <span class="nn">subprocess</span>
+ <span class="c1"># this will ensure the connection to the ssh.remote_host from where the tunnel</span>
+ <span class="c1"># is getting created</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+ <span class="n">tunnel_host</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">:</span><span class="si">{1}</span><span class="s2">:</span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">local_port</span><span class="p">,</span> <span class="n">remote_host</span><span class="p">,</span> <span class="n">remote_port</span><span class="p">)</span>
+
+ <span class="n">ssh_cmd</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"ssh"</span><span class="p">,</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">@</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">),</span>
+ <span class="s2">"-o"</span><span class="p">,</span> <span class="s2">"ControlMaster=no"</span><span class="p">,</span>
+ <span class="s2">"-o"</span><span class="p">,</span> <span class="s2">"UserKnownHostsFile=/dev/null"</span><span class="p">,</span>
+ <span class="s2">"-o"</span><span class="p">,</span> <span class="s2">"StrictHostKeyChecking=no"</span><span class="p">]</span>
+
+ <span class="n">ssh_tunnel_cmd</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"-L"</span><span class="p">,</span> <span class="n">tunnel_host</span><span class="p">,</span>
+ <span class="s2">"echo -n ready && cat"</span>
+ <span class="p">]</span>
+
+ <span class="n">ssh_cmd</span> <span class="o">+=</span> <span class="n">ssh_tunnel_cmd</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Creating tunnel with cmd: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">ssh_cmd</span><span class="p">)</span>
+
+ <span class="n">proc</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">Popen</span><span class="p">(</span><span class="n">ssh_cmd</span><span class="p">,</span>
+ <span class="n">stdin</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span>
+ <span class="n">stdout</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span>
+ <span class="n">close_fds</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="n">ready</span> <span class="o">=</span> <span class="n">proc</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
+ <span class="k">assert</span> <span class="n">ready</span> <span class="o">==</span> <span class="sa">b</span><span class="s2">"ready"</span><span class="p">,</span> \
+ <span class="s2">"Did not get 'ready' from remote, got '</span><span class="si">{0}</span><span class="s2">' instead"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">ready</span><span class="p">)</span>
+ <span class="k">yield</span>
+ <span class="n">proc</span><span class="o">.</span><span class="n">communicate</span><span class="p">()</span>
+ <span class="k">assert</span> <span class="n">proc</span><span class="o">.</span><span class="n">returncode</span> <span class="o">==</span> <span class="mi">0</span><span class="p">,</span> \
+ <span class="s2">"Tunnel process did unclean exit (returncode </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">proc</span><span class="o">.</span><span class="n">returncode</span><span class="p">)</span></div>
+
+ <span class="k">def</span> <span class="nf">__enter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span>
+
+ <span class="k">def</span> <span class="nf">__exit__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">exc_type</span><span class="p">,</span> <span class="n">exc_val</span><span class="p">,</span> <span class="n">exc_tb</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">close</span><span class="p">()</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/vertica_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/vertica_hook.html b/_modules/airflow/contrib/hooks/vertica_hook.html
new file mode 100644
index 0000000..355e266
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/vertica_hook.html
@@ -0,0 +1,288 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.vertica_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.vertica_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.vertica_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+
+<span class="kn">from</span> <span class="nn">vertica_python</span> <span class="k">import</span> <span class="n">connect</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="k">import</span> <span class="n">DbApiHook</span>
+
+
+<div class="viewcode-block" id="VerticaHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.vertica_hook.VerticaHook">[docs]</a><span class="k">class</span> <span class="nc">VerticaHook</span><span class="p">(</span><span class="n">DbApiHook</span><span class="p">):</span>
+ <span class="sd">'''</span>
+<span class="sd"> Interact with Vertica.</span>
+<span class="sd"> '''</span>
+
+ <span class="n">conn_name_attr</span> <span class="o">=</span> <span class="s1">'vertica_conn_id'</span>
+ <span class="n">default_conn_name</span> <span class="o">=</span> <span class="s1">'vertica_default'</span>
+ <span class="n">supports_autocommit</span> <span class="o">=</span> <span class="kc">True</span>
+
+<div class="viewcode-block" id="VerticaHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.vertica_hook.VerticaHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns verticaql connection object</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">vertica_conn_id</span><span class="p">)</span>
+ <span class="n">conn_config</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s2">"user"</span><span class="p">:</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span>
+ <span class="s2">"password"</span><span class="p">:</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span> <span class="ow">or</span> <span class="s1">''</span><span class="p">,</span>
+ <span class="s2">"database"</span><span class="p">:</span> <span class="n">conn</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span>
+ <span class="p">}</span>
+
+ <span class="n">conn_config</span><span class="p">[</span><span class="s2">"host"</span><span class="p">]</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span> <span class="ow">or</span> <span class="s1">'localhost'</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">:</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s2">"port"</span><span class="p">]</span> <span class="o">=</span> <span class="mi">5433</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s2">"port"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">)</span>
+
+ <span class="n">conn</span> <span class="o">=</span> <span class="n">connect</span><span class="p">(</span><span class="o">**</span><span class="n">conn_config</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">conn</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/hooks/wasb_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/wasb_hook.html b/_modules/airflow/contrib/hooks/wasb_hook.html
index 2ff6ba4..605fbf3 100644
--- a/_modules/airflow/contrib/hooks/wasb_hook.html
+++ b/_modules/airflow/contrib/hooks/wasb_hook.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,22 @@
<h1>Source code for airflow.contrib.hooks.wasb_hook</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="c1">#</span>
<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
@@ -190,11 +197,11 @@
<div class="viewcode-block" id="WasbHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.wasb_hook.WasbHook">[docs]</a><span class="k">class</span> <span class="nc">WasbHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Interacts with Azure Blob Storage through the wasb:// protocol.</span>
-<span class="sd"> </span>
+
<span class="sd"> Additional options passed in the 'extra' field of the connection will be</span>
<span class="sd"> passed to the `BlockBlockService()` constructor. For example, authenticate</span>
<span class="sd"> using a SAS token by adding {"sas_token": "YOUR_TOKEN"}.</span>
-<span class="sd"> </span>
+
<span class="sd"> :param wasb_conn_id: Reference to the wasb connection.</span>
<span class="sd"> :type wasb_conn_id: str</span>
<span class="sd"> """</span>
@@ -203,17 +210,17 @@
<span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span> <span class="o">=</span> <span class="n">wasb_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">connection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
- <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<div class="viewcode-block" id="WasbHook.get_conn"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.wasb_hook.WasbHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""Return the BlockBlobService object."""</span>
<span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
<span class="n">service_options</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span>
<span class="k">return</span> <span class="n">BlockBlobService</span><span class="p">(</span><span class="n">account_name</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span>
- <span class="n">account_key</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span> <span class="o">**</span><span class="n">service_options</span><span class="p">)</span>
+ <span class="n">account_key</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span> <span class="o">**</span><span class="n">service_options</span><span class="p">)</span></div>
- <span class="k">def</span> <span class="nf">check_for_blob</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">container_name</span><span class="p">,</span> <span class="n">blob_name</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+<div class="viewcode-block" id="WasbHook.check_for_blob"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.wasb_hook.WasbHook.check_for_blob">[docs]</a> <span class="k">def</span> <span class="nf">check_for_blob</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">container_name</span><span class="p">,</span> <span class="n">blob_name</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Check if a blob exists on Azure Blob Storage.</span>
-<span class="sd"> </span>
+
<span class="sd"> :param container_name: Name of the container.</span>
<span class="sd"> :type container_name: str</span>
<span class="sd"> :param blob_name: Name of the blob.</span>
@@ -224,12 +231,12 @@
<span class="sd"> :return: True if the blob exists, False otherwise.</span>
<span class="sd"> :rtype bool</span>
<span class="sd"> """</span>
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">container_name</span><span class="p">,</span> <span class="n">blob_name</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">container_name</span><span class="p">,</span> <span class="n">blob_name</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
- <span class="k">def</span> <span class="nf">check_for_prefix</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">container_name</span><span class="p">,</span> <span class="n">prefix</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+<div class="viewcode-block" id="WasbHook.check_for_prefix"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.wasb_hook.WasbHook.check_for_prefix">[docs]</a> <span class="k">def</span> <span class="nf">check_for_prefix</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">container_name</span><span class="p">,</span> <span class="n">prefix</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Check if a prefix exists on Azure Blob storage.</span>
-<span class="sd"> </span>
+
<span class="sd"> :param container_name: Name of the container.</span>
<span class="sd"> :type container_name: str</span>
<span class="sd"> :param prefix: Prefix of the blob.</span>
@@ -242,12 +249,12 @@
<span class="sd"> """</span>
<span class="n">matches</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">list_blobs</span><span class="p">(</span><span class="n">container_name</span><span class="p">,</span> <span class="n">prefix</span><span class="p">,</span>
<span class="n">num_results</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
- <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="n">matches</span><span class="p">))</span> <span class="o">></span> <span class="mi">0</span>
+ <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="n">matches</span><span class="p">))</span> <span class="o">></span> <span class="mi">0</span></div>
- <span class="k">def</span> <span class="nf">load_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">file_path</span><span class="p">,</span> <span class="n">container_name</span><span class="p">,</span> <span class="n">blob_name</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+<div class="viewcode-block" id="WasbHook.load_file"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.wasb_hook.WasbHook.load_file">[docs]</a> <span class="k">def</span> <span class="nf">load_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">file_path</span><span class="p">,</span> <span class="n">container_name</span><span class="p">,</span> <span class="n">blob_name</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Upload a file to Azure Blob Storage.</span>
-<span class="sd"> </span>
+
<span class="sd"> :param file_path: Path to the file to load.</span>
<span class="sd"> :type file_path: str</span>
<span class="sd"> :param container_name: Name of the container.</span>
@@ -260,12 +267,12 @@
<span class="sd"> """</span>
<span class="c1"># Reorder the argument order from airflow.hooks.S3_hook.load_file.</span>
<span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">create_blob_from_path</span><span class="p">(</span><span class="n">container_name</span><span class="p">,</span> <span class="n">blob_name</span><span class="p">,</span>
- <span class="n">file_path</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="n">file_path</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
- <span class="k">def</span> <span class="nf">load_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">string_data</span><span class="p">,</span> <span class="n">container_name</span><span class="p">,</span> <span class="n">blob_name</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+<div class="viewcode-block" id="WasbHook.load_string"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.wasb_hook.WasbHook.load_string">[docs]</a> <span class="k">def</span> <span class="nf">load_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">string_data</span><span class="p">,</span> <span class="n">container_name</span><span class="p">,</span> <span class="n">blob_name</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Upload a string to Azure Blob Storage.</span>
-<span class="sd"> </span>
+
<span class="sd"> :param string_data: String to load.</span>
<span class="sd"> :type string_data: str</span>
<span class="sd"> :param container_name: Name of the container.</span>
@@ -279,6 +286,39 @@
<span class="c1"># Reorder the argument order from airflow.hooks.S3_hook.load_string.</span>
<span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">create_blob_from_text</span><span class="p">(</span><span class="n">container_name</span><span class="p">,</span> <span class="n">blob_name</span><span class="p">,</span>
<span class="n">string_data</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="WasbHook.get_file"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.wasb_hook.WasbHook.get_file">[docs]</a> <span class="k">def</span> <span class="nf">get_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">file_path</span><span class="p">,</span> <span class="n">container_name</span><span class="p">,</span> <span class="n">blob_name</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Download a file from Azure Blob Storage.</span>
+
+<span class="sd"> :param file_path: Path to the file to download.</span>
+<span class="sd"> :type file_path: str</span>
+<span class="sd"> :param container_name: Name of the container.</span>
+<span class="sd"> :type container_name: str</span>
+<span class="sd"> :param blob_name: Name of the blob.</span>
+<span class="sd"> :type blob_name: str</span>
+<span class="sd"> :param kwargs: Optional keyword arguments that</span>
+<span class="sd"> `BlockBlobService.create_blob_from_path()` takes.</span>
+<span class="sd"> :type kwargs: object</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">get_blob_to_path</span><span class="p">(</span><span class="n">container_name</span><span class="p">,</span> <span class="n">blob_name</span><span class="p">,</span>
+ <span class="n">file_path</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="WasbHook.read_file"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.wasb_hook.WasbHook.read_file">[docs]</a> <span class="k">def</span> <span class="nf">read_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">container_name</span><span class="p">,</span> <span class="n">blob_name</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Read a file from Azure Blob Storage and return as a string.</span>
+
+<span class="sd"> :param container_name: Name of the container.</span>
+<span class="sd"> :type container_name: str</span>
+<span class="sd"> :param blob_name: Name of the blob.</span>
+<span class="sd"> :type blob_name: str</span>
+<span class="sd"> :param kwargs: Optional keyword arguments that</span>
+<span class="sd"> `BlockBlobService.create_blob_from_path()` takes.</span>
+<span class="sd"> :type kwargs: object</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">get_blob_to_text</span><span class="p">(</span><span class="n">container_name</span><span class="p">,</span>
+ <span class="n">blob_name</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span><span class="o">.</span><span class="n">content</span></div></div>
</pre></div>
</div>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/kubernetes/secret.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/kubernetes/secret.html b/_modules/airflow/contrib/kubernetes/secret.html
new file mode 100644
index 0000000..c5785ed
--- /dev/null
+++ b/_modules/airflow/contrib/kubernetes/secret.html
@@ -0,0 +1,276 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.kubernetes.secret — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.kubernetes.secret</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.kubernetes.secret</h1><div class="highlight"><pre>
+<span></span><span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+
+<div class="viewcode-block" id="Secret"><a class="viewcode-back" href="../../../../kubernetes.html#airflow.contrib.kubernetes.secret.Secret">[docs]</a><span class="k">class</span> <span class="nc">Secret</span><span class="p">:</span>
+ <span class="sd">"""Defines Kubernetes Secret Volume"""</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">deploy_type</span><span class="p">,</span> <span class="n">deploy_target</span><span class="p">,</span> <span class="n">secret</span><span class="p">,</span> <span class="n">key</span><span class="p">):</span>
+ <span class="sd">"""Initialize a Kubernetes Secret Object. Used to track requested secrets from</span>
+<span class="sd"> the user.</span>
+<span class="sd"> :param deploy_type: The type of secret deploy in Kubernetes, either `env` or</span>
+<span class="sd"> `volume`</span>
+<span class="sd"> :type deploy_type: ``str``</span>
+<span class="sd"> :param deploy_target: The environment variable when `deploy_type` `env` or</span>
+<span class="sd"> file path when `deploy_type` `volume` where expose secret</span>
+<span class="sd"> :type deploy_target: ``str``</span>
+<span class="sd"> :param secret: Name of the secrets object in Kubernetes</span>
+<span class="sd"> :type secret: ``str``</span>
+<span class="sd"> :param key: Key of the secret within the Kubernetes Secret</span>
+<span class="sd"> :type key: ``str``</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">deploy_type</span> <span class="o">=</span> <span class="n">deploy_type</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">deploy_target</span> <span class="o">=</span> <span class="n">deploy_target</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
+ <span class="k">if</span> <span class="n">deploy_type</span> <span class="o">==</span> <span class="s1">'volume'</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">deploy_target</span> <span class="o">=</span> <span class="n">deploy_target</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">secret</span> <span class="o">=</span> <span class="n">secret</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">key</span> <span class="o">=</span> <span class="n">key</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[09/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/postgres_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/postgres_operator.html b/_modules/airflow/operators/postgres_operator.html
new file mode 100644
index 0000000..dc0669e
--- /dev/null
+++ b/_modules/airflow/operators/postgres_operator.html
@@ -0,0 +1,297 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.operators.postgres_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.operators.postgres_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.operators.postgres_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.postgres_hook</span> <span class="k">import</span> <span class="n">PostgresHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="PostgresOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.postgres_operator.PostgresOperator">[docs]</a><span class="k">class</span> <span class="nc">PostgresOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Executes sql code in a specific Postgres database</span>
+
+<span class="sd"> :param postgres_conn_id: reference to a specific postgres database</span>
+<span class="sd"> :type postgres_conn_id: string</span>
+<span class="sd"> :param sql: the sql code to be executed</span>
+<span class="sd"> :type sql: Can receive a str representing a sql statement,</span>
+<span class="sd"> a list of str (sql statements), or reference to a template file.</span>
+<span class="sd"> Template reference are recognized by str ending in '.sql'</span>
+<span class="sd"> :param database: name of database which overwrite defined one in connection</span>
+<span class="sd"> :type database: string</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'sql'</span><span class="p">,)</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#ededed'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span>
+ <span class="n">postgres_conn_id</span><span class="o">=</span><span class="s1">'postgres_default'</span><span class="p">,</span> <span class="n">autocommit</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">database</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">PostgresOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">postgres_conn_id</span> <span class="o">=</span> <span class="n">postgres_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">autocommit</span> <span class="o">=</span> <span class="n">autocommit</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span> <span class="o">=</span> <span class="n">parameters</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">database</span> <span class="o">=</span> <span class="n">database</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">PostgresHook</span><span class="p">(</span><span class="n">postgres_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">postgres_conn_id</span><span class="p">,</span>
+ <span class="n">schema</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">database</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">autocommit</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">parameters</span><span class="p">)</span>
+ <span class="k">for</span> <span class="n">output</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">notices</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">output</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/presto_check_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/presto_check_operator.html b/_modules/airflow/operators/presto_check_operator.html
new file mode 100644
index 0000000..ceae902
--- /dev/null
+++ b/_modules/airflow/operators/presto_check_operator.html
@@ -0,0 +1,361 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.operators.presto_check_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.operators.presto_check_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.operators.presto_check_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.presto_hook</span> <span class="k">import</span> <span class="n">PrestoHook</span>
+<span class="kn">from</span> <span class="nn">airflow.operators.check_operator</span> <span class="k">import</span> <span class="n">CheckOperator</span><span class="p">,</span> <span class="n">ValueCheckOperator</span><span class="p">,</span> <span class="n">IntervalCheckOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="PrestoCheckOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.presto_check_operator.PrestoCheckOperator">[docs]</a><span class="k">class</span> <span class="nc">PrestoCheckOperator</span><span class="p">(</span><span class="n">CheckOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Performs checks against Presto. The ``PrestoCheckOperator`` expects</span>
+<span class="sd"> a sql query that will return a single row. Each value on that</span>
+<span class="sd"> first row is evaluated using python ``bool`` casting. If any of the</span>
+<span class="sd"> values return ``False`` the check is failed and errors out.</span>
+
+<span class="sd"> Note that Python bool casting evals the following as ``False``:</span>
+
+<span class="sd"> * ``False``</span>
+<span class="sd"> * ``0``</span>
+<span class="sd"> * Empty string (``""``)</span>
+<span class="sd"> * Empty list (``[]``)</span>
+<span class="sd"> * Empty dictionary or set (``{}``)</span>
+
+<span class="sd"> Given a query like ``SELECT COUNT(*) FROM foo``, it will fail only if</span>
+<span class="sd"> the count ``== 0``. You can craft much more complex query that could,</span>
+<span class="sd"> for instance, check that the table has the same number of rows as</span>
+<span class="sd"> the source table upstream, or that the count of today's partition is</span>
+<span class="sd"> greater than yesterday's partition, or that a set of metrics are less</span>
+<span class="sd"> than 3 standard deviation for the 7 day average.</span>
+
+<span class="sd"> This operator can be used as a data quality check in your pipeline, and</span>
+<span class="sd"> depending on where you put it in your DAG, you have the choice to</span>
+<span class="sd"> stop the critical path, preventing from</span>
+<span class="sd"> publishing dubious data, or on the side and receive email alterts</span>
+<span class="sd"> without stopping the progress of the DAG.</span>
+
+<span class="sd"> :param sql: the sql to be executed</span>
+<span class="sd"> :type sql: string</span>
+<span class="sd"> :param presto_conn_id: reference to the Presto database</span>
+<span class="sd"> :type presto_conn_id: string</span>
+<span class="sd"> """</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span>
+ <span class="n">presto_conn_id</span><span class="o">=</span><span class="s1">'presto_default'</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">PrestoCheckOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">sql</span><span class="o">=</span><span class="n">sql</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">presto_conn_id</span> <span class="o">=</span> <span class="n">presto_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span>
+
+ <span class="k">def</span> <span class="nf">get_db_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">PrestoHook</span><span class="p">(</span><span class="n">presto_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">presto_conn_id</span><span class="p">)</span></div>
+
+
+<div class="viewcode-block" id="PrestoValueCheckOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.presto_check_operator.PrestoValueCheckOperator">[docs]</a><span class="k">class</span> <span class="nc">PrestoValueCheckOperator</span><span class="p">(</span><span class="n">ValueCheckOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Performs a simple value check using sql code.</span>
+
+<span class="sd"> :param sql: the sql to be executed</span>
+<span class="sd"> :type sql: string</span>
+<span class="sd"> :param presto_conn_id: reference to the Presto database</span>
+<span class="sd"> :type presto_conn_id: string</span>
+<span class="sd"> """</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">pass_value</span><span class="p">,</span> <span class="n">tolerance</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">presto_conn_id</span><span class="o">=</span><span class="s1">'presto_default'</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">PrestoValueCheckOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+ <span class="n">sql</span><span class="o">=</span><span class="n">sql</span><span class="p">,</span> <span class="n">pass_value</span><span class="o">=</span><span class="n">pass_value</span><span class="p">,</span> <span class="n">tolerance</span><span class="o">=</span><span class="n">tolerance</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">presto_conn_id</span> <span class="o">=</span> <span class="n">presto_conn_id</span>
+
+ <span class="k">def</span> <span class="nf">get_db_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">PrestoHook</span><span class="p">(</span><span class="n">presto_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">presto_conn_id</span><span class="p">)</span></div>
+
+
+<div class="viewcode-block" id="PrestoIntervalCheckOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.presto_check_operator.PrestoIntervalCheckOperator">[docs]</a><span class="k">class</span> <span class="nc">PrestoIntervalCheckOperator</span><span class="p">(</span><span class="n">IntervalCheckOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Checks that the values of metrics given as SQL expressions are within</span>
+<span class="sd"> a certain tolerance of the ones from days_back before.</span>
+
+<span class="sd"> :param table: the table name</span>
+<span class="sd"> :type table: str</span>
+<span class="sd"> :param days_back: number of days between ds and the ds we want to check</span>
+<span class="sd"> against. Defaults to 7 days</span>
+<span class="sd"> :type days_back: int</span>
+<span class="sd"> :param metrics_threshold: a dictionary of ratios indexed by metrics</span>
+<span class="sd"> :type metrics_threshold: dict</span>
+<span class="sd"> :param presto_conn_id: reference to the Presto database</span>
+<span class="sd"> :type presto_conn_id: string</span>
+<span class="sd"> """</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">metrics_thresholds</span><span class="p">,</span>
+ <span class="n">date_filter_column</span><span class="o">=</span><span class="s1">'ds'</span><span class="p">,</span> <span class="n">days_back</span><span class="o">=-</span><span class="mi">7</span><span class="p">,</span>
+ <span class="n">presto_conn_id</span><span class="o">=</span><span class="s1">'presto_default'</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">PrestoIntervalCheckOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+ <span class="n">table</span><span class="o">=</span><span class="n">table</span><span class="p">,</span> <span class="n">metrics_thresholds</span><span class="o">=</span><span class="n">metrics_thresholds</span><span class="p">,</span>
+ <span class="n">date_filter_column</span><span class="o">=</span><span class="n">date_filter_column</span><span class="p">,</span> <span class="n">days_back</span><span class="o">=</span><span class="n">days_back</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">presto_conn_id</span> <span class="o">=</span> <span class="n">presto_conn_id</span>
+
+ <span class="k">def</span> <span class="nf">get_db_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">PrestoHook</span><span class="p">(</span><span class="n">presto_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">presto_conn_id</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/presto_to_mysql.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/presto_to_mysql.html b/_modules/airflow/operators/presto_to_mysql.html
new file mode 100644
index 0000000..0f27e92
--- /dev/null
+++ b/_modules/airflow/operators/presto_to_mysql.html
@@ -0,0 +1,314 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.operators.presto_to_mysql — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.operators.presto_to_mysql</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.operators.presto_to_mysql</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.presto_hook</span> <span class="k">import</span> <span class="n">PrestoHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="k">import</span> <span class="n">MySqlHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="PrestoToMySqlTransfer"><a class="viewcode-back" href="../../../code.html#airflow.operators.presto_to_mysql.PrestoToMySqlTransfer">[docs]</a><span class="k">class</span> <span class="nc">PrestoToMySqlTransfer</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Moves data from Presto to MySQL, note that for now the data is loaded</span>
+<span class="sd"> into memory before being pushed to MySQL, so this operator should</span>
+<span class="sd"> be used for smallish amount of data.</span>
+
+<span class="sd"> :param sql: SQL query to execute against Presto</span>
+<span class="sd"> :type sql: str</span>
+<span class="sd"> :param mysql_table: target MySQL table, use dot notation to target a</span>
+<span class="sd"> specific database</span>
+<span class="sd"> :type mysql_table: str</span>
+<span class="sd"> :param mysql_conn_id: source mysql connection</span>
+<span class="sd"> :type mysql_conn_id: str</span>
+<span class="sd"> :param presto_conn_id: source presto connection</span>
+<span class="sd"> :type presto_conn_id: str</span>
+<span class="sd"> :param mysql_preoperator: sql statement to run against mysql prior to</span>
+<span class="sd"> import, typically use to truncate of delete in place of the data</span>
+<span class="sd"> coming in, allowing the task to be idempotent (running the task</span>
+<span class="sd"> twice won't double load data)</span>
+<span class="sd"> :type mysql_preoperator: str</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'sql'</span><span class="p">,</span> <span class="s1">'mysql_table'</span><span class="p">,</span> <span class="s1">'mysql_preoperator'</span><span class="p">)</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#a0e08c'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">sql</span><span class="p">,</span>
+ <span class="n">mysql_table</span><span class="p">,</span>
+ <span class="n">presto_conn_id</span><span class="o">=</span><span class="s1">'presto_default'</span><span class="p">,</span>
+ <span class="n">mysql_conn_id</span><span class="o">=</span><span class="s1">'mysql_default'</span><span class="p">,</span>
+ <span class="n">mysql_preoperator</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">PrestoToMySqlTransfer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mysql_table</span> <span class="o">=</span> <span class="n">mysql_table</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mysql_conn_id</span> <span class="o">=</span> <span class="n">mysql_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mysql_preoperator</span> <span class="o">=</span> <span class="n">mysql_preoperator</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">presto_conn_id</span> <span class="o">=</span> <span class="n">presto_conn_id</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">presto</span> <span class="o">=</span> <span class="n">PrestoHook</span><span class="p">(</span><span class="n">presto_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">presto_conn_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Extracting data from Presto: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
+ <span class="n">results</span> <span class="o">=</span> <span class="n">presto</span><span class="o">.</span><span class="n">get_records</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
+
+ <span class="n">mysql</span> <span class="o">=</span> <span class="n">MySqlHook</span><span class="p">(</span><span class="n">mysql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mysql_conn_id</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mysql_preoperator</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Running MySQL preoperator"</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mysql_preoperator</span><span class="p">)</span>
+ <span class="n">mysql</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mysql_preoperator</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Inserting rows into MySQL"</span><span class="p">)</span>
+ <span class="n">mysql</span><span class="o">.</span><span class="n">insert_rows</span><span class="p">(</span><span class="n">table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mysql_table</span><span class="p">,</span> <span class="n">rows</span><span class="o">=</span><span class="n">results</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[12/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/bash_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/bash_operator.html b/_modules/airflow/operators/bash_operator.html
new file mode 100644
index 0000000..d6d7dc6
--- /dev/null
+++ b/_modules/airflow/operators/bash_operator.html
@@ -0,0 +1,376 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.operators.bash_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.operators.bash_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.operators.bash_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">bytes</span>
+<span class="kn">import</span> <span class="nn">os</span>
+<span class="kn">import</span> <span class="nn">signal</span>
+<span class="kn">from</span> <span class="nn">subprocess</span> <span class="k">import</span> <span class="n">Popen</span><span class="p">,</span> <span class="n">STDOUT</span><span class="p">,</span> <span class="n">PIPE</span>
+<span class="kn">from</span> <span class="nn">tempfile</span> <span class="k">import</span> <span class="n">gettempdir</span><span class="p">,</span> <span class="n">NamedTemporaryFile</span>
+
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">configuration</span> <span class="k">as</span> <span class="n">conf</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.file</span> <span class="k">import</span> <span class="n">TemporaryDirectory</span>
+
+
+<span class="c1"># These variables are required in cases when BashOperator tasks use airflow specific code,</span>
+<span class="c1"># e.g. they import packages in the airflow context and the possibility of impersonation</span>
+<span class="c1"># gives not guarantee that these variables are available in the impersonated environment.</span>
+<span class="c1"># Hence, we need to propagate them in the Bash script used as a wrapper of commands in</span>
+<span class="c1"># this BashOperator.</span>
+<span class="n">PYTHONPATH_VAR</span> <span class="o">=</span> <span class="s1">'PYTHONPATH'</span>
+<span class="n">AIRFLOW_HOME_VAR</span> <span class="o">=</span> <span class="s1">'AIRFLOW_HOME'</span>
+
+
+<div class="viewcode-block" id="BashOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.bash_operator.BashOperator">[docs]</a><span class="k">class</span> <span class="nc">BashOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Execute a Bash script, command or set of commands.</span>
+
+<span class="sd"> :param bash_command: The command, set of commands or reference to a</span>
+<span class="sd"> bash script (must be '.sh') to be executed.</span>
+<span class="sd"> :type bash_command: string</span>
+<span class="sd"> :param xcom_push: If xcom_push is True, the last line written to stdout</span>
+<span class="sd"> will also be pushed to an XCom when the bash command completes.</span>
+<span class="sd"> :type xcom_push: bool</span>
+<span class="sd"> :param env: If env is not None, it must be a mapping that defines the</span>
+<span class="sd"> environment variables for the new process; these are used instead</span>
+<span class="sd"> of inheriting the current process environment, which is the default</span>
+<span class="sd"> behavior. (templated)</span>
+<span class="sd"> :type env: dict</span>
+<span class="sd"> :type output_encoding: output encoding of bash command</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bash_command'</span><span class="p">,</span> <span class="s1">'env'</span><span class="p">)</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sh'</span><span class="p">,</span> <span class="s1">'.bash'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f0ede4'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">bash_command</span><span class="p">,</span>
+ <span class="n">xcom_push</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">env</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">output_encoding</span><span class="o">=</span><span class="s1">'utf-8'</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+
+ <span class="nb">super</span><span class="p">(</span><span class="n">BashOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bash_command</span> <span class="o">=</span> <span class="n">bash_command</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">env</span> <span class="o">=</span> <span class="n">env</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">xcom_push_flag</span> <span class="o">=</span> <span class="n">xcom_push</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">output_encoding</span> <span class="o">=</span> <span class="n">output_encoding</span>
+
+<div class="viewcode-block" id="BashOperator.execute"><a class="viewcode-back" href="../../../code.html#airflow.operators.bash_operator.BashOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Execute the bash command in a temporary directory</span>
+<span class="sd"> which will be cleaned afterwards</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Tmp dir root location: </span><span class="se">\n</span><span class="s2"> </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">gettempdir</span><span class="p">())</span>
+
+ <span class="n">airflow_home_value</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="n">AIRFLOW_HOME_VAR</span><span class="p">)</span>
+ <span class="n">pythonpath_value</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">PYTHONPATH_VAR</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
+
+ <span class="n">bash_command</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'export </span><span class="si">{}</span><span class="s1">=</span><span class="si">{}</span><span class="s1">; '</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">AIRFLOW_HOME_VAR</span><span class="p">,</span> <span class="n">airflow_home_value</span><span class="p">)</span> <span class="o">+</span>
+ <span class="s1">'export </span><span class="si">{}</span><span class="s1">=</span><span class="si">{}</span><span class="s1">; '</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">PYTHONPATH_VAR</span><span class="p">,</span> <span class="n">pythonpath_value</span><span class="p">)</span> <span class="o">+</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bash_command</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">lineage_data</span> <span class="o">=</span> <span class="n">bash_command</span>
+
+ <span class="k">with</span> <span class="n">TemporaryDirectory</span><span class="p">(</span><span class="n">prefix</span><span class="o">=</span><span class="s1">'airflowtmp'</span><span class="p">)</span> <span class="k">as</span> <span class="n">tmp_dir</span><span class="p">:</span>
+ <span class="k">with</span> <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="nb">dir</span><span class="o">=</span><span class="n">tmp_dir</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
+
+ <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="nb">bytes</span><span class="p">(</span><span class="n">bash_command</span><span class="p">,</span> <span class="s1">'utf_8'</span><span class="p">))</span>
+ <span class="n">f</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
+ <span class="n">fname</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">name</span>
+ <span class="n">script_location</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="n">fname</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s2">"Temporary script location: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
+ <span class="n">script_location</span>
+ <span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">pre_exec</span><span class="p">():</span>
+ <span class="c1"># Restore default signal disposition and invoke setsid</span>
+ <span class="k">for</span> <span class="n">sig</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'SIGPIPE'</span><span class="p">,</span> <span class="s1">'SIGXFZ'</span><span class="p">,</span> <span class="s1">'SIGXFSZ'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">signal</span><span class="p">,</span> <span class="n">sig</span><span class="p">):</span>
+ <span class="n">signal</span><span class="o">.</span><span class="n">signal</span><span class="p">(</span><span class="nb">getattr</span><span class="p">(</span><span class="n">signal</span><span class="p">,</span> <span class="n">sig</span><span class="p">),</span> <span class="n">signal</span><span class="o">.</span><span class="n">SIG_DFL</span><span class="p">)</span>
+ <span class="n">os</span><span class="o">.</span><span class="n">setsid</span><span class="p">()</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Running command: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">bash_command</span><span class="p">)</span>
+ <span class="n">sp</span> <span class="o">=</span> <span class="n">Popen</span><span class="p">(</span>
+ <span class="p">[</span><span class="s1">'bash'</span><span class="p">,</span> <span class="n">fname</span><span class="p">],</span>
+ <span class="n">stdout</span><span class="o">=</span><span class="n">PIPE</span><span class="p">,</span> <span class="n">stderr</span><span class="o">=</span><span class="n">STDOUT</span><span class="p">,</span>
+ <span class="n">cwd</span><span class="o">=</span><span class="n">tmp_dir</span><span class="p">,</span> <span class="n">env</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">env</span><span class="p">,</span>
+ <span class="n">preexec_fn</span><span class="o">=</span><span class="n">pre_exec</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">sp</span> <span class="o">=</span> <span class="n">sp</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Output:"</span><span class="p">)</span>
+ <span class="n">line</span> <span class="o">=</span> <span class="s1">''</span>
+ <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="nb">iter</span><span class="p">(</span><span class="n">sp</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">readline</span><span class="p">,</span> <span class="sa">b</span><span class="s1">''</span><span class="p">):</span>
+ <span class="n">line</span> <span class="o">=</span> <span class="n">line</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">output_encoding</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">line</span><span class="p">)</span>
+ <span class="n">sp</span><span class="o">.</span><span class="n">wait</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s2">"Command exited with return code </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
+ <span class="n">sp</span><span class="o">.</span><span class="n">returncode</span>
+ <span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">sp</span><span class="o">.</span><span class="n">returncode</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Bash command failed"</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">xcom_push_flag</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">line</span></div>
+
+ <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Sending SIGTERM signal to bash process group'</span><span class="p">)</span>
+ <span class="n">os</span><span class="o">.</span><span class="n">killpg</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">getpgid</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sp</span><span class="o">.</span><span class="n">pid</span><span class="p">),</span> <span class="n">signal</span><span class="o">.</span><span class="n">SIGTERM</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/check_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/check_operator.html b/_modules/airflow/operators/check_operator.html
new file mode 100644
index 0000000..7e367ba
--- /dev/null
+++ b/_modules/airflow/operators/check_operator.html
@@ -0,0 +1,497 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.operators.check_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.operators.check_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.operators.check_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">zip</span>
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="CheckOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.check_operator.CheckOperator">[docs]</a><span class="k">class</span> <span class="nc">CheckOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Performs checks against a db. The ``CheckOperator`` expects</span>
+<span class="sd"> a sql query that will return a single row. Each value on that</span>
+<span class="sd"> first row is evaluated using python ``bool`` casting. If any of the</span>
+<span class="sd"> values return ``False`` the check is failed and errors out.</span>
+
+<span class="sd"> Note that Python bool casting evals the following as ``False``:</span>
+
+<span class="sd"> * ``False``</span>
+<span class="sd"> * ``0``</span>
+<span class="sd"> * Empty string (``""``)</span>
+<span class="sd"> * Empty list (``[]``)</span>
+<span class="sd"> * Empty dictionary or set (``{}``)</span>
+
+<span class="sd"> Given a query like ``SELECT COUNT(*) FROM foo``, it will fail only if</span>
+<span class="sd"> the count ``== 0``. You can craft much more complex query that could,</span>
+<span class="sd"> for instance, check that the table has the same number of rows as</span>
+<span class="sd"> the source table upstream, or that the count of today's partition is</span>
+<span class="sd"> greater than yesterday's partition, or that a set of metrics are less</span>
+<span class="sd"> than 3 standard deviation for the 7 day average.</span>
+
+<span class="sd"> This operator can be used as a data quality check in your pipeline, and</span>
+<span class="sd"> depending on where you put it in your DAG, you have the choice to</span>
+<span class="sd"> stop the critical path, preventing from</span>
+<span class="sd"> publishing dubious data, or on the side and receive email alerts</span>
+<span class="sd"> without stopping the progress of the DAG.</span>
+
+<span class="sd"> Note that this is an abstract class and get_db_hook</span>
+<span class="sd"> needs to be defined. Whereas a get_db_hook is hook that gets a</span>
+<span class="sd"> single record from an external source.</span>
+
+<span class="sd"> :param sql: the sql to be executed</span>
+<span class="sd"> :type sql: string</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'sql'</span><span class="p">,)</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.hql'</span><span class="p">,</span> <span class="s1">'.sql'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#fff7e6'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span>
+ <span class="n">conn_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">CheckOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span> <span class="o">=</span> <span class="n">conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing SQL check: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
+ <span class="n">records</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_db_hook</span><span class="p">()</span><span class="o">.</span><span class="n">get_first</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Record: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">records</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">records</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"The query returned None"</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="ow">not</span> <span class="nb">all</span><span class="p">([</span><span class="nb">bool</span><span class="p">(</span><span class="n">r</span><span class="p">)</span> <span class="k">for</span> <span class="n">r</span> <span class="ow">in</span> <span class="n">records</span><span class="p">]):</span>
+ <span class="n">exceptstr</span> <span class="o">=</span> <span class="s2">"Test failed.</span><span class="se">\n</span><span class="s2">Query:</span><span class="se">\n</span><span class="si">{q}</span><span class="se">\n</span><span class="s2">Results:</span><span class="se">\n</span><span class="si">{r!s}</span><span class="s2">"</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="n">exceptstr</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">q</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">,</span> <span class="n">r</span><span class="o">=</span><span class="n">records</span><span class="p">))</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Success."</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">get_db_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">BaseHook</span><span class="o">.</span><span class="n">get_hook</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span></div>
+
+
+<span class="k">def</span> <span class="nf">_convert_to_float_if_possible</span><span class="p">(</span><span class="n">s</span><span class="p">):</span>
+ <span class="sd">'''</span>
+<span class="sd"> A small helper function to convert a string to a numeric value</span>
+<span class="sd"> if appropriate</span>
+
+<span class="sd"> :param s: the string to be converted</span>
+<span class="sd"> :type s: str</span>
+<span class="sd"> '''</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">ret</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">s</span><span class="p">)</span>
+ <span class="k">except</span> <span class="p">(</span><span class="ne">ValueError</span><span class="p">,</span> <span class="ne">TypeError</span><span class="p">):</span>
+ <span class="n">ret</span> <span class="o">=</span> <span class="n">s</span>
+ <span class="k">return</span> <span class="n">ret</span>
+
+
+<div class="viewcode-block" id="ValueCheckOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.check_operator.ValueCheckOperator">[docs]</a><span class="k">class</span> <span class="nc">ValueCheckOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Performs a simple value check using sql code.</span>
+
+<span class="sd"> Note that this is an abstract class and get_db_hook</span>
+<span class="sd"> needs to be defined. Whereas a get_db_hook is hook that gets a</span>
+<span class="sd"> single record from an external source.</span>
+
+<span class="sd"> :param sql: the sql to be executed</span>
+<span class="sd"> :type sql: string</span>
+<span class="sd"> """</span>
+
+ <span class="n">__mapper_args__</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'polymorphic_identity'</span><span class="p">:</span> <span class="s1">'ValueCheckOperator'</span>
+ <span class="p">}</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'sql'</span><span class="p">,</span> <span class="s1">'pass_value'</span><span class="p">,)</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.hql'</span><span class="p">,</span> <span class="s1">'.sql'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#fff7e6'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">pass_value</span><span class="p">,</span> <span class="n">tolerance</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">conn_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">ValueCheckOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span> <span class="o">=</span> <span class="n">conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">pass_value</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">pass_value</span><span class="p">)</span>
+ <span class="n">tol</span> <span class="o">=</span> <span class="n">_convert_to_float_if_possible</span><span class="p">(</span><span class="n">tolerance</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">tol</span> <span class="o">=</span> <span class="n">tol</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">tol</span><span class="p">,</span> <span class="nb">float</span><span class="p">)</span> <span class="k">else</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">has_tolerance</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">tol</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing SQL check: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
+ <span class="n">records</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_db_hook</span><span class="p">()</span><span class="o">.</span><span class="n">get_first</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">records</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"The query returned None"</span><span class="p">)</span>
+
+ <span class="n">pass_value_conv</span> <span class="o">=</span> <span class="n">_convert_to_float_if_possible</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">pass_value</span><span class="p">)</span>
+ <span class="n">is_numeric_value_check</span> <span class="o">=</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">pass_value_conv</span><span class="p">,</span> <span class="nb">float</span><span class="p">)</span>
+
+ <span class="n">tolerance_pct_str</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="k">if</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tol</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">):</span>
+ <span class="n">tolerance_pct_str</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tol</span> <span class="o">*</span> <span class="mi">100</span><span class="p">)</span> <span class="o">+</span> <span class="s1">'%'</span>
+
+ <span class="n">except_temp</span> <span class="o">=</span> <span class="p">(</span><span class="s2">"Test failed.</span><span class="se">\n</span><span class="s2">Pass value:</span><span class="si">{pass_value_conv}</span><span class="se">\n</span><span class="s2">"</span>
+ <span class="s2">"Tolerance:</span><span class="si">{tolerance_pct_str}</span><span class="se">\n</span><span class="s2">"</span>
+ <span class="s2">"Query:</span><span class="se">\n</span><span class="si">{self.sql}</span><span class="se">\n</span><span class="s2">Results:</span><span class="se">\n</span><span class="si">{records!s}</span><span class="s2">"</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">is_numeric_value_check</span><span class="p">:</span>
+ <span class="n">tests</span> <span class="o">=</span> <span class="p">[</span><span class="nb">str</span><span class="p">(</span><span class="n">r</span><span class="p">)</span> <span class="o">==</span> <span class="n">pass_value_conv</span> <span class="k">for</span> <span class="n">r</span> <span class="ow">in</span> <span class="n">records</span><span class="p">]</span>
+ <span class="k">elif</span> <span class="n">is_numeric_value_check</span><span class="p">:</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">num_rec</span> <span class="o">=</span> <span class="p">[</span><span class="nb">float</span><span class="p">(</span><span class="n">r</span><span class="p">)</span> <span class="k">for</span> <span class="n">r</span> <span class="ow">in</span> <span class="n">records</span><span class="p">]</span>
+ <span class="k">except</span> <span class="p">(</span><span class="ne">ValueError</span><span class="p">,</span> <span class="ne">TypeError</span><span class="p">)</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="n">cvestr</span> <span class="o">=</span> <span class="s2">"Converting a result to float failed.</span><span class="se">\n</span><span class="s2">"</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="n">cvestr</span> <span class="o">+</span> <span class="n">except_temp</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">has_tolerance</span><span class="p">:</span>
+ <span class="n">tests</span> <span class="o">=</span> <span class="p">[</span>
+ <span class="n">pass_value_conv</span> <span class="o">*</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">tol</span><span class="p">)</span> <span class="o"><=</span>
+ <span class="n">r</span> <span class="o"><=</span> <span class="n">pass_value_conv</span> <span class="o">*</span> <span class="p">(</span><span class="mi">1</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">tol</span><span class="p">)</span>
+ <span class="k">for</span> <span class="n">r</span> <span class="ow">in</span> <span class="n">num_rec</span><span class="p">]</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">tests</span> <span class="o">=</span> <span class="p">[</span><span class="n">r</span> <span class="o">==</span> <span class="n">pass_value_conv</span> <span class="k">for</span> <span class="n">r</span> <span class="ow">in</span> <span class="n">num_rec</span><span class="p">]</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="nb">all</span><span class="p">(</span><span class="n">tests</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="n">except_temp</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
+
+ <span class="k">def</span> <span class="nf">get_db_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">BaseHook</span><span class="o">.</span><span class="n">get_hook</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span></div>
+
+
+<div class="viewcode-block" id="IntervalCheckOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.check_operator.IntervalCheckOperator">[docs]</a><span class="k">class</span> <span class="nc">IntervalCheckOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Checks that the values of metrics given as SQL expressions are within</span>
+<span class="sd"> a certain tolerance of the ones from days_back before.</span>
+
+<span class="sd"> Note that this is an abstract class and get_db_hook</span>
+<span class="sd"> needs to be defined. Whereas a get_db_hook is hook that gets a</span>
+<span class="sd"> single record from an external source.</span>
+
+<span class="sd"> :param table: the table name</span>
+<span class="sd"> :type table: str</span>
+<span class="sd"> :param days_back: number of days between ds and the ds we want to check</span>
+<span class="sd"> against. Defaults to 7 days</span>
+<span class="sd"> :type days_back: int</span>
+<span class="sd"> :param metrics_threshold: a dictionary of ratios indexed by metrics</span>
+<span class="sd"> :type metrics_threshold: dict</span>
+<span class="sd"> """</span>
+
+ <span class="n">__mapper_args__</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'polymorphic_identity'</span><span class="p">:</span> <span class="s1">'IntervalCheckOperator'</span>
+ <span class="p">}</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'sql1'</span><span class="p">,</span> <span class="s1">'sql2'</span><span class="p">)</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.hql'</span><span class="p">,</span> <span class="s1">'.sql'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#fff7e6'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">metrics_thresholds</span><span class="p">,</span>
+ <span class="n">date_filter_column</span><span class="o">=</span><span class="s1">'ds'</span><span class="p">,</span> <span class="n">days_back</span><span class="o">=-</span><span class="mi">7</span><span class="p">,</span>
+ <span class="n">conn_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">IntervalCheckOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">table</span> <span class="o">=</span> <span class="n">table</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">metrics_thresholds</span> <span class="o">=</span> <span class="n">metrics_thresholds</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">metrics_sorted</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">metrics_thresholds</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">date_filter_column</span> <span class="o">=</span> <span class="n">date_filter_column</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">days_back</span> <span class="o">=</span> <span class="o">-</span><span class="nb">abs</span><span class="p">(</span><span class="n">days_back</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span> <span class="o">=</span> <span class="n">conn_id</span>
+ <span class="n">sqlexp</span> <span class="o">=</span> <span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">metrics_sorted</span><span class="p">)</span>
+ <span class="n">sqlt</span> <span class="o">=</span> <span class="p">(</span><span class="s2">"SELECT </span><span class="si">{sqlexp}</span><span class="s2"> FROM </span><span class="si">{table}</span><span class="s2">"</span>
+ <span class="s2">" WHERE </span><span class="si">{date_filter_column}</span><span class="s2">="</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sql1</span> <span class="o">=</span> <span class="n">sqlt</span> <span class="o">+</span> <span class="s2">"'{{ ds }}'"</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sql2</span> <span class="o">=</span> <span class="n">sqlt</span> <span class="o">+</span> <span class="s2">"'{{ macros.ds_add(ds, "</span><span class="o">+</span><span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">days_back</span><span class="p">)</span><span class="o">+</span><span class="s2">") }}'"</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_db_hook</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing SQL check: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql2</span><span class="p">)</span>
+ <span class="n">row2</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_first</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql2</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing SQL check: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql1</span><span class="p">)</span>
+ <span class="n">row1</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_first</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql1</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">row2</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"The query </span><span class="si">{q}</span><span class="s2"> returned None"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">q</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">sql2</span><span class="p">))</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">row1</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"The query </span><span class="si">{q}</span><span class="s2"> returned None"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">q</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">sql1</span><span class="p">))</span>
+ <span class="n">current</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">metrics_sorted</span><span class="p">,</span> <span class="n">row1</span><span class="p">))</span>
+ <span class="n">reference</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">metrics_sorted</span><span class="p">,</span> <span class="n">row2</span><span class="p">))</span>
+ <span class="n">ratios</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="n">test_results</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="n">rlog</span> <span class="o">=</span> <span class="s2">"Ratio for </span><span class="si">{0}</span><span class="s2">: </span><span class="si">{1}</span><span class="s2"> </span><span class="se">\n</span><span class="s2"> Ratio threshold : </span><span class="si">{2}</span><span class="s2">"</span>
+ <span class="n">fstr</span> <span class="o">=</span> <span class="s2">"'</span><span class="si">{k}</span><span class="s2">' check failed. </span><span class="si">{r}</span><span class="s2"> is above </span><span class="si">{tr}</span><span class="s2">"</span>
+ <span class="n">estr</span> <span class="o">=</span> <span class="s2">"The following tests have failed:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{0}</span><span class="s2">"</span>
+ <span class="n">countstr</span> <span class="o">=</span> <span class="s2">"The following </span><span class="si">{j}</span><span class="s2"> tests out of </span><span class="si">{n}</span><span class="s2"> failed:"</span>
+ <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">metrics_sorted</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">current</span><span class="p">[</span><span class="n">m</span><span class="p">]</span> <span class="o">==</span> <span class="mi">0</span> <span class="ow">or</span> <span class="n">reference</span><span class="p">[</span><span class="n">m</span><span class="p">]</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="n">ratio</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">ratio</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="nb">max</span><span class="p">(</span><span class="n">current</span><span class="p">[</span><span class="n">m</span><span class="p">],</span> <span class="n">reference</span><span class="p">[</span><span class="n">m</span><span class="p">]))</span> <span class="o">/</span> \
+ <span class="nb">min</span><span class="p">(</span><span class="n">current</span><span class="p">[</span><span class="n">m</span><span class="p">],</span> <span class="n">reference</span><span class="p">[</span><span class="n">m</span><span class="p">])</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">rlog</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">m</span><span class="p">,</span> <span class="n">ratio</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">metrics_thresholds</span><span class="p">[</span><span class="n">m</span><span class="p">]))</span>
+ <span class="n">ratios</span><span class="p">[</span><span class="n">m</span><span class="p">]</span> <span class="o">=</span> <span class="n">ratio</span>
+ <span class="n">test_results</span><span class="p">[</span><span class="n">m</span><span class="p">]</span> <span class="o">=</span> <span class="n">ratio</span> <span class="o"><</span> <span class="bp">self</span><span class="o">.</span><span class="n">metrics_thresholds</span><span class="p">[</span><span class="n">m</span><span class="p">]</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="nb">all</span><span class="p">(</span><span class="n">test_results</span><span class="o">.</span><span class="n">values</span><span class="p">()):</span>
+ <span class="n">failed_tests</span> <span class="o">=</span> <span class="p">[</span><span class="n">it</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">it</span> <span class="ow">in</span> <span class="n">test_results</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">it</span><span class="p">[</span><span class="mi">1</span><span class="p">]]</span>
+ <span class="n">j</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">failed_tests</span><span class="p">)</span>
+ <span class="n">n</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">metrics_sorted</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">countstr</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
+ <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">failed_tests</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
+ <span class="n">fstr</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">k</span><span class="o">=</span><span class="n">k</span><span class="p">,</span> <span class="n">r</span><span class="o">=</span><span class="n">ratios</span><span class="p">[</span><span class="n">k</span><span class="p">],</span> <span class="n">tr</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metrics_thresholds</span><span class="p">[</span><span class="n">k</span><span class="p">])</span>
+ <span class="p">)</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="n">estr</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="s2">", "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">failed_tests</span><span class="p">)))</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"All tests have passed"</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">get_db_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">BaseHook</span><span class="o">.</span><span class="n">get_hook</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/dagrun_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/dagrun_operator.html b/_modules/airflow/operators/dagrun_operator.html
new file mode 100644
index 0000000..b77ac68
--- /dev/null
+++ b/_modules/airflow/operators/dagrun_operator.html
@@ -0,0 +1,317 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.operators.dagrun_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.operators.dagrun_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.operators.dagrun_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span><span class="p">,</span> <span class="n">DagBag</span>
+<span class="kn">from</span> <span class="nn">airflow.utils</span> <span class="k">import</span> <span class="n">timezone</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.db</span> <span class="k">import</span> <span class="n">create_session</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span>
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">settings</span>
+
+
+<span class="k">class</span> <span class="nc">DagRunOrder</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">run_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">payload</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">run_id</span> <span class="o">=</span> <span class="n">run_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">payload</span> <span class="o">=</span> <span class="n">payload</span>
+
+
+<div class="viewcode-block" id="TriggerDagRunOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.dagrun_operator.TriggerDagRunOperator">[docs]</a><span class="k">class</span> <span class="nc">TriggerDagRunOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Triggers a DAG run for a specified ``dag_id``</span>
+
+<span class="sd"> :param trigger_dag_id: the dag_id to trigger</span>
+<span class="sd"> :type trigger_dag_id: str</span>
+<span class="sd"> :param python_callable: a reference to a python function that will be</span>
+<span class="sd"> called while passing it the ``context`` object and a placeholder</span>
+<span class="sd"> object ``obj`` for your callable to fill and return if you want</span>
+<span class="sd"> a DagRun created. This ``obj`` object contains a ``run_id`` and</span>
+<span class="sd"> ``payload`` attribute that you can modify in your function.</span>
+<span class="sd"> The ``run_id`` should be a unique identifier for that DAG run, and</span>
+<span class="sd"> the payload has to be a picklable object that will be made available</span>
+<span class="sd"> to your tasks while executing that DAG run. Your function header</span>
+<span class="sd"> should look like ``def foo(context, dag_run_obj):``</span>
+<span class="sd"> :type python_callable: python callable</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">()</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">()</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#ffefeb'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">trigger_dag_id</span><span class="p">,</span>
+ <span class="n">python_callable</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">TriggerDagRunOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">python_callable</span> <span class="o">=</span> <span class="n">python_callable</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">trigger_dag_id</span> <span class="o">=</span> <span class="n">trigger_dag_id</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">dro</span> <span class="o">=</span> <span class="n">DagRunOrder</span><span class="p">(</span><span class="n">run_id</span><span class="o">=</span><span class="s1">'trig__'</span> <span class="o">+</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span><span class="o">.</span><span class="n">isoformat</span><span class="p">())</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">python_callable</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">dro</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">python_callable</span><span class="p">(</span><span class="n">context</span><span class="p">,</span> <span class="n">dro</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">dro</span><span class="p">:</span>
+ <span class="k">with</span> <span class="n">create_session</span><span class="p">()</span> <span class="k">as</span> <span class="n">session</span><span class="p">:</span>
+ <span class="n">dbag</span> <span class="o">=</span> <span class="n">DagBag</span><span class="p">(</span><span class="n">settings</span><span class="o">.</span><span class="n">DAGS_FOLDER</span><span class="p">)</span>
+ <span class="n">trigger_dag</span> <span class="o">=</span> <span class="n">dbag</span><span class="o">.</span><span class="n">get_dag</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">trigger_dag_id</span><span class="p">)</span>
+ <span class="n">dr</span> <span class="o">=</span> <span class="n">trigger_dag</span><span class="o">.</span><span class="n">create_dagrun</span><span class="p">(</span>
+ <span class="n">run_id</span><span class="o">=</span><span class="n">dro</span><span class="o">.</span><span class="n">run_id</span><span class="p">,</span>
+ <span class="n">state</span><span class="o">=</span><span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">,</span>
+ <span class="n">conf</span><span class="o">=</span><span class="n">dro</span><span class="o">.</span><span class="n">payload</span><span class="p">,</span>
+ <span class="n">external_trigger</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Creating DagRun </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">dr</span><span class="p">)</span>
+ <span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">dr</span><span class="p">)</span>
+ <span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Criteria not met, moving on"</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[14/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/hooks/slack_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/hooks/slack_hook.html b/_modules/airflow/hooks/slack_hook.html
new file mode 100644
index 0000000..f331546
--- /dev/null
+++ b/_modules/airflow/hooks/slack_hook.html
@@ -0,0 +1,296 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.hooks.slack_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.hooks.slack_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.hooks.slack_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">slackclient</span> <span class="k">import</span> <span class="n">SlackClient</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+
+
+<div class="viewcode-block" id="SlackHook"><a class="viewcode-back" href="../../../code.html#airflow.hooks.slack_hook.SlackHook">[docs]</a><span class="k">class</span> <span class="nc">SlackHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with Slack, using slackclient library.</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">token</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">slack_conn_id</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Takes both Slack API token directly and connection that has Slack API token.</span>
+
+<span class="sd"> If both supplied, Slack API token will be used.</span>
+
+<span class="sd"> :param token: Slack API token</span>
+<span class="sd"> :type token: string</span>
+<span class="sd"> :param slack_conn_id: connection that has Slack API token in the password field</span>
+<span class="sd"> :type slack_conn_id: string</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">token</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">__get_token</span><span class="p">(</span><span class="n">token</span><span class="p">,</span> <span class="n">slack_conn_id</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">__get_token</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">token</span><span class="p">,</span> <span class="n">slack_conn_id</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">token</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">token</span>
+ <span class="k">elif</span> <span class="n">slack_conn_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">slack_conn_id</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">conn</span><span class="p">,</span> <span class="s1">'password'</span><span class="p">,</span> <span class="kc">None</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Missing token(password) in Slack connection'</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Cannot get token: No valid Slack token nor slack_conn_id supplied.'</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">call</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">method</span><span class="p">,</span> <span class="n">api_params</span><span class="p">):</span>
+ <span class="n">sc</span> <span class="o">=</span> <span class="n">SlackClient</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">token</span><span class="p">)</span>
+ <span class="n">rc</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">api_call</span><span class="p">(</span><span class="n">method</span><span class="p">,</span> <span class="o">**</span><span class="n">api_params</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">rc</span><span class="p">[</span><span class="s1">'ok'</span><span class="p">]:</span>
+ <span class="n">msg</span> <span class="o">=</span> <span class="s2">"Slack API call failed (</span><span class="si">{}</span><span class="s2">)"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">rc</span><span class="p">[</span><span class="s1">'error'</span><span class="p">])</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/hooks/sqlite_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/hooks/sqlite_hook.html b/_modules/airflow/hooks/sqlite_hook.html
new file mode 100644
index 0000000..c3027d1
--- /dev/null
+++ b/_modules/airflow/hooks/sqlite_hook.html
@@ -0,0 +1,276 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.hooks.sqlite_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.hooks.sqlite_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.hooks.sqlite_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">sqlite3</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="k">import</span> <span class="n">DbApiHook</span>
+
+
+<div class="viewcode-block" id="SqliteHook"><a class="viewcode-back" href="../../../code.html#airflow.hooks.sqlite_hook.SqliteHook">[docs]</a><span class="k">class</span> <span class="nc">SqliteHook</span><span class="p">(</span><span class="n">DbApiHook</span><span class="p">):</span>
+
+ <span class="sd">"""</span>
+<span class="sd"> Interact with SQLite.</span>
+<span class="sd"> """</span>
+
+ <span class="n">conn_name_attr</span> <span class="o">=</span> <span class="s1">'sqlite_conn_id'</span>
+ <span class="n">default_conn_name</span> <span class="o">=</span> <span class="s1">'sqlite_default'</span>
+ <span class="n">supports_autocommit</span> <span class="o">=</span> <span class="kc">False</span>
+
+<div class="viewcode-block" id="SqliteHook.get_conn"><a class="viewcode-back" href="../../../code.html#airflow.hooks.sqlite_hook.SqliteHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns a sqlite connection object</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sqlite_conn_id</span><span class="p">)</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="n">sqlite3</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">conn</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/hooks/webhdfs_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/hooks/webhdfs_hook.html b/_modules/airflow/hooks/webhdfs_hook.html
new file mode 100644
index 0000000..772e94a
--- /dev/null
+++ b/_modules/airflow/hooks/webhdfs_hook.html
@@ -0,0 +1,344 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.hooks.webhdfs_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.hooks.webhdfs_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.hooks.webhdfs_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">configuration</span>
+
+<span class="kn">from</span> <span class="nn">hdfs</span> <span class="k">import</span> <span class="n">InsecureClient</span><span class="p">,</span> <span class="n">HdfsError</span>
+
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+<span class="n">_kerberos_security_mode</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"core"</span><span class="p">,</span> <span class="s2">"security"</span><span class="p">)</span> <span class="o">==</span> <span class="s2">"kerberos"</span>
+<span class="k">if</span> <span class="n">_kerberos_security_mode</span><span class="p">:</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="kn">from</span> <span class="nn">hdfs.ext.kerberos</span> <span class="k">import</span> <span class="n">KerberosClient</span>
+ <span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
+ <span class="n">log</span> <span class="o">=</span> <span class="n">LoggingMixin</span><span class="p">()</span><span class="o">.</span><span class="n">log</span>
+ <span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Could not load the Kerberos extension for the WebHDFSHook."</span><span class="p">)</span>
+ <span class="k">raise</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+
+
+<span class="k">class</span> <span class="nc">AirflowWebHDFSHookException</span><span class="p">(</span><span class="n">AirflowException</span><span class="p">):</span>
+ <span class="k">pass</span>
+
+
+<div class="viewcode-block" id="WebHDFSHook"><a class="viewcode-back" href="../../../code.html#airflow.hooks.webhdfs_hook.WebHDFSHook">[docs]</a><span class="k">class</span> <span class="nc">WebHDFSHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with HDFS. This class is a wrapper around the hdfscli library.</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">webhdfs_conn_id</span><span class="o">=</span><span class="s1">'webhdfs_default'</span><span class="p">,</span> <span class="n">proxy_user</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">webhdfs_conn_id</span> <span class="o">=</span> <span class="n">webhdfs_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">proxy_user</span> <span class="o">=</span> <span class="n">proxy_user</span>
+
+<div class="viewcode-block" id="WebHDFSHook.get_conn"><a class="viewcode-back" href="../../../code.html#airflow.hooks.webhdfs_hook.WebHDFSHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns a hdfscli InsecureClient object.</span>
+<span class="sd"> """</span>
+ <span class="n">nn_connections</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connections</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">webhdfs_conn_id</span><span class="p">)</span>
+ <span class="k">for</span> <span class="n">nn</span> <span class="ow">in</span> <span class="n">nn_connections</span><span class="p">:</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'Trying namenode </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">host</span><span class="p">)</span>
+ <span class="n">connection_str</span> <span class="o">=</span> <span class="s1">'http://</span><span class="si">{nn.host}</span><span class="s1">:</span><span class="si">{nn.port}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">nn</span><span class="o">=</span><span class="n">nn</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">_kerberos_security_mode</span><span class="p">:</span>
+ <span class="n">client</span> <span class="o">=</span> <span class="n">KerberosClient</span><span class="p">(</span><span class="n">connection_str</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">proxy_user</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">proxy_user</span> <span class="ow">or</span> <span class="n">nn</span><span class="o">.</span><span class="n">login</span>
+ <span class="n">client</span> <span class="o">=</span> <span class="n">InsecureClient</span><span class="p">(</span><span class="n">connection_str</span><span class="p">,</span> <span class="n">user</span><span class="o">=</span><span class="n">proxy_user</span><span class="p">)</span>
+ <span class="n">client</span><span class="o">.</span><span class="n">status</span><span class="p">(</span><span class="s1">'/'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'Using namenode </span><span class="si">%s</span><span class="s1"> for hook'</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">host</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">client</span>
+ <span class="k">except</span> <span class="n">HdfsError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
+ <span class="s2">"Read operation on namenode </span><span class="si">{nn.host}</span><span class="s2"> failed with error: </span><span class="si">{e}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
+ <span class="p">)</span>
+ <span class="n">nn_hosts</span> <span class="o">=</span> <span class="p">[</span><span class="n">c</span><span class="o">.</span><span class="n">host</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">nn_connections</span><span class="p">]</span>
+ <span class="n">no_nn_error</span> <span class="o">=</span> <span class="s2">"Read operations failed on the namenodes below:</span><span class="se">\n</span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">nn_hosts</span><span class="p">))</span>
+ <span class="k">raise</span> <span class="n">AirflowWebHDFSHookException</span><span class="p">(</span><span class="n">no_nn_error</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="WebHDFSHook.check_for_path"><a class="viewcode-back" href="../../../code.html#airflow.hooks.webhdfs_hook.WebHDFSHook.check_for_path">[docs]</a> <span class="k">def</span> <span class="nf">check_for_path</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hdfs_path</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Check for the existence of a path in HDFS by querying FileStatus.</span>
+<span class="sd"> """</span>
+ <span class="n">c</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="k">return</span> <span class="nb">bool</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">status</span><span class="p">(</span><span class="n">hdfs_path</span><span class="p">,</span> <span class="n">strict</span><span class="o">=</span><span class="kc">False</span><span class="p">))</span></div>
+
+<div class="viewcode-block" id="WebHDFSHook.load_file"><a class="viewcode-back" href="../../../code.html#airflow.hooks.webhdfs_hook.WebHDFSHook.load_file">[docs]</a> <span class="k">def</span> <span class="nf">load_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">source</span><span class="p">,</span> <span class="n">destination</span><span class="p">,</span> <span class="n">overwrite</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">parallelism</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Uploads a file to HDFS</span>
+
+<span class="sd"> :param source: Local path to file or folder. If a folder, all the files</span>
+<span class="sd"> inside of it will be uploaded (note that this implies that folders empty</span>
+<span class="sd"> of files will not be created remotely).</span>
+<span class="sd"> :type source: str</span>
+<span class="sd"> :param destination: PTarget HDFS path. If it already exists and is a</span>
+<span class="sd"> directory, files will be uploaded inside.</span>
+<span class="sd"> :type destination: str</span>
+<span class="sd"> :param overwrite: Overwrite any existing file or directory.</span>
+<span class="sd"> :type overwrite: bool</span>
+<span class="sd"> :param parallelism: Number of threads to use for parallelization. A value of</span>
+<span class="sd"> `0` (or negative) uses as many threads as there are files.</span>
+<span class="sd"> :type parallelism: int</span>
+<span class="sd"> :param \*\*kwargs: Keyword arguments forwarded to :meth:`upload`.</span>
+
+
+<span class="sd"> """</span>
+ <span class="n">c</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">c</span><span class="o">.</span><span class="n">upload</span><span class="p">(</span><span class="n">hdfs_path</span><span class="o">=</span><span class="n">destination</span><span class="p">,</span>
+ <span class="n">local_path</span><span class="o">=</span><span class="n">source</span><span class="p">,</span>
+ <span class="n">overwrite</span><span class="o">=</span><span class="n">overwrite</span><span class="p">,</span>
+ <span class="n">n_threads</span><span class="o">=</span><span class="n">parallelism</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Uploaded file </span><span class="si">%s</span><span class="s2"> to </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">source</span><span class="p">,</span> <span class="n">destination</span><span class="p">)</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/macros.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/macros.html b/_modules/airflow/macros.html
index 48eb480..ce55a46 100644
--- a/_modules/airflow/macros.html
+++ b/_modules/airflow/macros.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,22 @@
<h1>Source code for airflow.macros</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="kn">from</span> <span class="nn">__future__</span> <span class="k">import</span> <span class="n">absolute_import</span>
<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/macros/hive.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/macros/hive.html b/_modules/airflow/macros/hive.html
index ba316f6..a308be0 100644
--- a/_modules/airflow/macros/hive.html
+++ b/_modules/airflow/macros/hive.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
</ul>
@@ -171,25 +173,30 @@
<h1>Source code for airflow.macros.hive</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="kn">import</span> <span class="nn">datetime</span>
<div class="viewcode-block" id="max_partition"><a class="viewcode-back" href="../../../code.html#airflow.macros.hive.max_partition">[docs]</a><span class="k">def</span> <span class="nf">max_partition</span><span class="p">(</span>
- <span class="n">table</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="s2">"default"</span><span class="p">,</span> <span class="n">field</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="nb">filter</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">table</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="s2">"default"</span><span class="p">,</span> <span class="n">field</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">filter_map</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">metastore_conn_id</span><span class="o">=</span><span class="s1">'metastore_default'</span><span class="p">):</span>
- <span class="sd">'''</span>
+ <span class="sd">"""</span>
<span class="sd"> Gets the max partition for a table.</span>
<span class="sd"> :param schema: The hive schema the table lives in</span>
@@ -198,24 +205,27 @@
<span class="sd"> notation as in "my_database.my_table", if a dot is found,</span>
<span class="sd"> the schema param is disregarded</span>
<span class="sd"> :type table: string</span>
-<span class="sd"> :param hive_conn_id: The hive connection you are interested in.</span>
+<span class="sd"> :param metastore_conn_id: The hive connection you are interested in.</span>
<span class="sd"> If your default is set you don't need to use this parameter.</span>
-<span class="sd"> :type hive_conn_id: string</span>
-<span class="sd"> :param filter: filter on a subset of partition as in</span>
-<span class="sd"> `sub_part='specific_value'`</span>
-<span class="sd"> :type filter: string</span>
+<span class="sd"> :type metastore_conn_id: string</span>
+<span class="sd"> :param filter_map: partition_key:partition_value map used for partition filtering,</span>
+<span class="sd"> e.g. {'key1': 'value1', 'key2': 'value2'}.</span>
+<span class="sd"> Only partitions matching all partition_key:partition_value</span>
+<span class="sd"> pairs will be considered as candidates of max partition.</span>
+<span class="sd"> :type filter_map: map</span>
<span class="sd"> :param field: the field to get the max value from. If there's only</span>
<span class="sd"> one partition field, this will be inferred</span>
+<span class="sd"> :type field: str</span>
<span class="sd"> >>> max_partition('airflow.static_babynames_partitioned')</span>
<span class="sd"> '2015-01-01'</span>
-<span class="sd"> '''</span>
+<span class="sd"> """</span>
<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="k">import</span> <span class="n">HiveMetastoreHook</span>
<span class="k">if</span> <span class="s1">'.'</span> <span class="ow">in</span> <span class="n">table</span><span class="p">:</span>
<span class="n">schema</span><span class="p">,</span> <span class="n">table</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)</span>
<span class="n">hh</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">(</span><span class="n">metastore_conn_id</span><span class="o">=</span><span class="n">metastore_conn_id</span><span class="p">)</span>
<span class="k">return</span> <span class="n">hh</span><span class="o">.</span><span class="n">max_partition</span><span class="p">(</span>
- <span class="n">schema</span><span class="o">=</span><span class="n">schema</span><span class="p">,</span> <span class="n">table_name</span><span class="o">=</span><span class="n">table</span><span class="p">,</span> <span class="n">field</span><span class="o">=</span><span class="n">field</span><span class="p">,</span> <span class="nb">filter</span><span class="o">=</span><span class="nb">filter</span><span class="p">)</span></div>
+ <span class="n">schema</span><span class="o">=</span><span class="n">schema</span><span class="p">,</span> <span class="n">table_name</span><span class="o">=</span><span class="n">table</span><span class="p">,</span> <span class="n">field</span><span class="o">=</span><span class="n">field</span><span class="p">,</span> <span class="n">filter_map</span><span class="o">=</span><span class="n">filter_map</span><span class="p">)</span></div>
<span class="k">def</span> <span class="nf">_closest_date</span><span class="p">(</span><span class="n">target_dt</span><span class="p">,</span> <span class="n">date_list</span><span class="p">,</span> <span class="n">before_target</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
[11/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/docker_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/docker_operator.html b/_modules/airflow/operators/docker_operator.html
deleted file mode 100644
index 10ff19a..0000000
--- a/_modules/airflow/operators/docker_operator.html
+++ /dev/null
@@ -1,469 +0,0 @@
-
-
-<!DOCTYPE html>
-<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
-<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
-<head>
- <meta charset="utf-8">
-
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
-
- <title>airflow.operators.docker_operator — Airflow Documentation</title>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
- <link rel="up" title="Module code" href="../../index.html"/>
-
-
- <script src="../../../_static/js/modernizr.min.js"></script>
-
-</head>
-
-<body class="wy-body-for-nav" role="document">
-
-
- <div class="wy-grid-for-nav">
-
-
- <nav data-toggle="wy-nav-shift" class="wy-nav-side">
- <div class="wy-side-scroll">
- <div class="wy-side-nav-search">
-
-
-
- <a href="../../../index.html" class="icon icon-home"> Airflow
-
-
-
- </a>
-
-
-
-
-
-
-
-<div role="search">
- <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
- <input type="text" name="q" placeholder="Search docs" />
- <input type="hidden" name="check_keywords" value="yes" />
- <input type="hidden" name="area" value="default" />
- </form>
-</div>
-
-
- </div>
-
- <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
-
-
-
-
-
-
- <ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../configuration.html">Configuration</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
-</ul>
-
-
-
- </div>
- </div>
- </nav>
-
- <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
-
-
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
-
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="../../../index.html">Airflow</a>
-
- </nav>
-
-
-
- <div class="wy-nav-content">
- <div class="rst-content">
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-<div role="navigation" aria-label="breadcrumbs navigation">
-
- <ul class="wy-breadcrumbs">
-
- <li><a href="../../../index.html">Docs</a> »</li>
-
- <li><a href="../../index.html">Module code</a> »</li>
-
- <li>airflow.operators.docker_operator</li>
-
-
- <li class="wy-breadcrumbs-aside">
-
-
-
- </li>
-
- </ul>
-
-
- <hr/>
-</div>
- <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
- <div itemprop="articleBody">
-
- <h1>Source code for airflow.operators.docker_operator</h1><div class="highlight"><pre>
-<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
-<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
-
-<span class="kn">import</span> <span class="nn">json</span>
-
-<span class="kn">from</span> <span class="nn">airflow.hooks.docker_hook</span> <span class="k">import</span> <span class="n">DockerHook</span>
-<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
-<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.file</span> <span class="k">import</span> <span class="n">TemporaryDirectory</span>
-<span class="kn">from</span> <span class="nn">docker</span> <span class="k">import</span> <span class="n">Client</span><span class="p">,</span> <span class="n">tls</span>
-<span class="kn">import</span> <span class="nn">ast</span>
-
-
-<div class="viewcode-block" id="DockerOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.docker_operator.DockerOperator">[docs]</a><span class="k">class</span> <span class="nc">DockerOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Execute a command inside a docker container.</span>
-
-<span class="sd"> A temporary directory is created on the host and mounted into a container to allow storing files</span>
-<span class="sd"> that together exceed the default disk size of 10GB in a container. The path to the mounted</span>
-<span class="sd"> directory can be accessed via the environment variable ``AIRFLOW_TMP_DIR``.</span>
-
-<span class="sd"> If a login to a private registry is required prior to pulling the image, a</span>
-<span class="sd"> Docker connection needs to be configured in Airflow and the connection ID</span>
-<span class="sd"> be provided with the parameter ``docker_conn_id``.</span>
-
-<span class="sd"> :param image: Docker image from which to create the container.</span>
-<span class="sd"> :type image: str</span>
-<span class="sd"> :param api_version: Remote API version. Set to ``auto`` to automatically</span>
-<span class="sd"> detect the server's version.</span>
-<span class="sd"> :type api_version: str</span>
-<span class="sd"> :param command: Command to be run in the container.</span>
-<span class="sd"> :type command: str or list</span>
-<span class="sd"> :param cpus: Number of CPUs to assign to the container.</span>
-<span class="sd"> This value gets multiplied with 1024. See</span>
-<span class="sd"> https://docs.docker.com/engine/reference/run/#cpu-share-constraint</span>
-<span class="sd"> :type cpus: float</span>
-<span class="sd"> :param docker_url: URL of the host running the docker daemon.</span>
-<span class="sd"> Default is unix://var/run/docker.sock</span>
-<span class="sd"> :type docker_url: str</span>
-<span class="sd"> :param environment: Environment variables to set in the container.</span>
-<span class="sd"> :type environment: dict</span>
-<span class="sd"> :param force_pull: Pull the docker image on every run. Default is false.</span>
-<span class="sd"> :type force_pull: bool</span>
-<span class="sd"> :param mem_limit: Maximum amount of memory the container can use. Either a float value, which</span>
-<span class="sd"> represents the limit in bytes, or a string like ``128m`` or ``1g``.</span>
-<span class="sd"> :type mem_limit: float or str</span>
-<span class="sd"> :param network_mode: Network mode for the container.</span>
-<span class="sd"> :type network_mode: str</span>
-<span class="sd"> :param tls_ca_cert: Path to a PEM-encoded certificate authority to secure the docker connection.</span>
-<span class="sd"> :type tls_ca_cert: str</span>
-<span class="sd"> :param tls_client_cert: Path to the PEM-encoded certificate used to authenticate docker client.</span>
-<span class="sd"> :type tls_client_cert: str</span>
-<span class="sd"> :param tls_client_key: Path to the PEM-encoded key used to authenticate docker client.</span>
-<span class="sd"> :type tls_client_key: str</span>
-<span class="sd"> :param tls_hostname: Hostname to match against the docker server certificate or False to</span>
-<span class="sd"> disable the check.</span>
-<span class="sd"> :type tls_hostname: str or bool</span>
-<span class="sd"> :param tls_ssl_version: Version of SSL to use when communicating with docker daemon.</span>
-<span class="sd"> :type tls_ssl_version: str</span>
-<span class="sd"> :param tmp_dir: Mount point inside the container to a temporary directory created on the host by</span>
-<span class="sd"> the operator. The path is also made available via the environment variable</span>
-<span class="sd"> ``AIRFLOW_TMP_DIR`` inside the container.</span>
-<span class="sd"> :type tmp_dir: str</span>
-<span class="sd"> :param user: Default user inside the docker container.</span>
-<span class="sd"> :type user: int or str</span>
-<span class="sd"> :param volumes: List of volumes to mount into the container, e.g.</span>
-<span class="sd"> ``['/host/path:/container/path', '/host/path2:/container/path2:ro']``.</span>
-<span class="sd"> :param working_dir: Working directory to set on the container (equivalent to the -w switch</span>
-<span class="sd"> the docker client)</span>
-<span class="sd"> :type working_dir: str</span>
-<span class="sd"> :param xcom_push: Does the stdout will be pushed to the next step using XCom.</span>
-<span class="sd"> The default is False.</span>
-<span class="sd"> :type xcom_push: bool</span>
-<span class="sd"> :param xcom_all: Push all the stdout or just the last line. The default is False (last line).</span>
-<span class="sd"> :type xcom_all: bool</span>
-<span class="sd"> :param docker_conn_id: ID of the Airflow connection to use</span>
-<span class="sd"> :type docker_conn_id: str</span>
-<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'command'</span><span class="p">,)</span>
- <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sh'</span><span class="p">,</span> <span class="s1">'.bash'</span><span class="p">,)</span>
-
- <span class="nd">@apply_defaults</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
- <span class="bp">self</span><span class="p">,</span>
- <span class="n">image</span><span class="p">,</span>
- <span class="n">api_version</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">command</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">cpus</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span>
- <span class="n">docker_url</span><span class="o">=</span><span class="s1">'unix://var/run/docker.sock'</span><span class="p">,</span>
- <span class="n">environment</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">force_pull</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">mem_limit</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">network_mode</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">tls_ca_cert</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">tls_client_cert</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">tls_client_key</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">tls_hostname</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">tls_ssl_version</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">tmp_dir</span><span class="o">=</span><span class="s1">'/tmp/airflow'</span><span class="p">,</span>
- <span class="n">user</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">volumes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">working_dir</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">xcom_push</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">xcom_all</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">docker_conn_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="o">*</span><span class="n">args</span><span class="p">,</span>
- <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
-
- <span class="nb">super</span><span class="p">(</span><span class="n">DockerOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">api_version</span> <span class="o">=</span> <span class="n">api_version</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">command</span> <span class="o">=</span> <span class="n">command</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">cpus</span> <span class="o">=</span> <span class="n">cpus</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">docker_url</span> <span class="o">=</span> <span class="n">docker_url</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">environment</span> <span class="o">=</span> <span class="n">environment</span> <span class="ow">or</span> <span class="p">{}</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">force_pull</span> <span class="o">=</span> <span class="n">force_pull</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">image</span> <span class="o">=</span> <span class="n">image</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">mem_limit</span> <span class="o">=</span> <span class="n">mem_limit</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">network_mode</span> <span class="o">=</span> <span class="n">network_mode</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">tls_ca_cert</span> <span class="o">=</span> <span class="n">tls_ca_cert</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">tls_client_cert</span> <span class="o">=</span> <span class="n">tls_client_cert</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">tls_client_key</span> <span class="o">=</span> <span class="n">tls_client_key</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">tls_hostname</span> <span class="o">=</span> <span class="n">tls_hostname</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">tls_ssl_version</span> <span class="o">=</span> <span class="n">tls_ssl_version</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">tmp_dir</span> <span class="o">=</span> <span class="n">tmp_dir</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">user</span> <span class="o">=</span> <span class="n">user</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">volumes</span> <span class="o">=</span> <span class="n">volumes</span> <span class="ow">or</span> <span class="p">[]</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">working_dir</span> <span class="o">=</span> <span class="n">working_dir</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">xcom_push_flag</span> <span class="o">=</span> <span class="n">xcom_push</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">xcom_all</span> <span class="o">=</span> <span class="n">xcom_all</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">docker_conn_id</span> <span class="o">=</span> <span class="n">docker_conn_id</span>
-
- <span class="bp">self</span><span class="o">.</span><span class="n">cli</span> <span class="o">=</span> <span class="kc">None</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">container</span> <span class="o">=</span> <span class="kc">None</span>
-
- <span class="k">def</span> <span class="nf">get_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="k">return</span> <span class="n">DockerHook</span><span class="p">(</span>
- <span class="n">docker_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">docker_conn_id</span><span class="p">,</span>
- <span class="n">base_url</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">base_url</span><span class="p">,</span>
- <span class="n">version</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">api_version</span><span class="p">,</span>
- <span class="n">tls</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">__get_tls_config</span><span class="p">()</span>
- <span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Starting docker container from image </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">image</span><span class="p">)</span>
-
- <span class="n">tls_config</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">__get_tls_config</span><span class="p">()</span>
-
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">docker_conn_id</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">cli</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">cli</span> <span class="o">=</span> <span class="n">Client</span><span class="p">(</span>
- <span class="n">base_url</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">docker_url</span><span class="p">,</span>
- <span class="n">version</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">api_version</span><span class="p">,</span>
- <span class="n">tls</span><span class="o">=</span><span class="n">tls_config</span>
- <span class="p">)</span>
-
- <span class="k">if</span> <span class="s1">':'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">image</span><span class="p">:</span>
- <span class="n">image</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">image</span> <span class="o">+</span> <span class="s1">':latest'</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">image</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">image</span>
-
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">force_pull</span> <span class="ow">or</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cli</span><span class="o">.</span><span class="n">images</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="n">image</span><span class="p">))</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Pulling docker image </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">image</span><span class="p">)</span>
- <span class="k">for</span> <span class="n">l</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cli</span><span class="o">.</span><span class="n">pull</span><span class="p">(</span><span class="n">image</span><span class="p">,</span> <span class="n">stream</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
- <span class="n">output</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">l</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">))</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"</span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">output</span><span class="p">[</span><span class="s1">'status'</span><span class="p">])</span>
-
- <span class="n">cpu_shares</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="nb">round</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cpus</span> <span class="o">*</span> <span class="mi">1024</span><span class="p">))</span>
-
- <span class="k">with</span> <span class="n">TemporaryDirectory</span><span class="p">(</span><span class="n">prefix</span><span class="o">=</span><span class="s1">'airflowtmp'</span><span class="p">)</span> <span class="k">as</span> <span class="n">host_tmp_dir</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">environment</span><span class="p">[</span><span class="s1">'AIRFLOW_TMP_DIR'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">tmp_dir</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">volumes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s1">'</span><span class="si">{0}</span><span class="s1">:</span><span class="si">{1}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">host_tmp_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">tmp_dir</span><span class="p">))</span>
-
- <span class="bp">self</span><span class="o">.</span><span class="n">container</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cli</span><span class="o">.</span><span class="n">create_container</span><span class="p">(</span>
- <span class="n">command</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">get_command</span><span class="p">(),</span>
- <span class="n">cpu_shares</span><span class="o">=</span><span class="n">cpu_shares</span><span class="p">,</span>
- <span class="n">environment</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">environment</span><span class="p">,</span>
- <span class="n">host_config</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cli</span><span class="o">.</span><span class="n">create_host_config</span><span class="p">(</span><span class="n">binds</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">volumes</span><span class="p">,</span>
- <span class="n">network_mode</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">network_mode</span><span class="p">),</span>
- <span class="n">image</span><span class="o">=</span><span class="n">image</span><span class="p">,</span>
- <span class="n">mem_limit</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mem_limit</span><span class="p">,</span>
- <span class="n">user</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">user</span><span class="p">,</span>
- <span class="n">working_dir</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">working_dir</span>
- <span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">cli</span><span class="o">.</span><span class="n">start</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">container</span><span class="p">[</span><span class="s1">'Id'</span><span class="p">])</span>
-
- <span class="n">line</span> <span class="o">=</span> <span class="s1">''</span>
- <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cli</span><span class="o">.</span><span class="n">logs</span><span class="p">(</span><span class="n">container</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">container</span><span class="p">[</span><span class="s1">'Id'</span><span class="p">],</span> <span class="n">stream</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
- <span class="n">line</span> <span class="o">=</span> <span class="n">line</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
- <span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">line</span><span class="p">,</span> <span class="s1">'decode'</span><span class="p">):</span>
- <span class="n">line</span> <span class="o">=</span> <span class="n">line</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">line</span><span class="p">)</span>
-
- <span class="n">exit_code</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cli</span><span class="o">.</span><span class="n">wait</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">container</span><span class="p">[</span><span class="s1">'Id'</span><span class="p">])</span>
- <span class="k">if</span> <span class="n">exit_code</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'docker container failed'</span><span class="p">)</span>
-
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">xcom_push_flag</span><span class="p">:</span>
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">cli</span><span class="o">.</span><span class="n">logs</span><span class="p">(</span><span class="n">container</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">container</span><span class="p">[</span><span class="s1">'Id'</span><span class="p">])</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">xcom_all</span> <span class="k">else</span> <span class="nb">str</span><span class="p">(</span><span class="n">line</span><span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">get_command</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">command</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">command</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">'['</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
- <span class="n">commands</span> <span class="o">=</span> <span class="n">ast</span><span class="o">.</span><span class="n">literal_eval</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">command</span><span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">commands</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">command</span>
- <span class="k">return</span> <span class="n">commands</span>
-
- <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cli</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Stopping docker container'</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">cli</span><span class="o">.</span><span class="n">stop</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">container</span><span class="p">[</span><span class="s1">'Id'</span><span class="p">])</span>
-
- <span class="k">def</span> <span class="nf">__get_tls_config</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="n">tls_config</span> <span class="o">=</span> <span class="kc">None</span>
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">tls_ca_cert</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">tls_client_cert</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">tls_client_key</span><span class="p">:</span>
- <span class="n">tls_config</span> <span class="o">=</span> <span class="n">tls</span><span class="o">.</span><span class="n">TLSConfig</span><span class="p">(</span>
- <span class="n">ca_cert</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">tls_ca_cert</span><span class="p">,</span>
- <span class="n">client_cert</span><span class="o">=</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tls_client_cert</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">tls_client_key</span><span class="p">),</span>
- <span class="n">verify</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
- <span class="n">ssl_version</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">tls_ssl_version</span><span class="p">,</span>
- <span class="n">assert_hostname</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">tls_hostname</span>
- <span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">docker_url</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">docker_url</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'tcp://'</span><span class="p">,</span> <span class="s1">'https://'</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">tls_config</span></div>
-</pre></div>
-
- </div>
- <div class="articleComments">
-
- </div>
- </div>
- <footer>
-
-
- <hr/>
-
- <div role="contentinfo">
- <p>
-
- </p>
- </div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
-
-</footer>
-
- </div>
- </div>
-
- </section>
-
- </div>
-
-
-
-
-
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../',
- VERSION:'',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../_static/doctools.js"></script>
-
-
-
-
-
- <script type="text/javascript" src="../../../_static/js/theme.js"></script>
-
-
-
-
- <script type="text/javascript">
- jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
- });
- </script>
-
-
-</body>
-</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/dummy_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/dummy_operator.html b/_modules/airflow/operators/dummy_operator.html
new file mode 100644
index 0000000..e730cbe
--- /dev/null
+++ b/_modules/airflow/operators/dummy_operator.html
@@ -0,0 +1,273 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.operators.dummy_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.operators.dummy_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.operators.dummy_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="DummyOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.dummy_operator.DummyOperator">[docs]</a><span class="k">class</span> <span class="nc">DummyOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Operator that does literally nothing. It can be used to group tasks in a</span>
+<span class="sd"> DAG.</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">()</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#e8f7e4'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">DummyOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="k">pass</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/email_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/email_operator.html b/_modules/airflow/operators/email_operator.html
new file mode 100644
index 0000000..6a5980a
--- /dev/null
+++ b/_modules/airflow/operators/email_operator.html
@@ -0,0 +1,313 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.operators.email_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.operators.email_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.operators.email_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.email</span> <span class="k">import</span> <span class="n">send_email</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="EmailOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.email_operator.EmailOperator">[docs]</a><span class="k">class</span> <span class="nc">EmailOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Sends an email.</span>
+
+<span class="sd"> :param to: list of emails to send the email to</span>
+<span class="sd"> :type to: list or string (comma or semicolon delimited)</span>
+<span class="sd"> :param subject: subject line for the email (templated)</span>
+<span class="sd"> :type subject: string</span>
+<span class="sd"> :param html_content: content of the email (templated), html markup</span>
+<span class="sd"> is allowed</span>
+<span class="sd"> :type html_content: string</span>
+<span class="sd"> :param files: file names to attach in email</span>
+<span class="sd"> :type files: list</span>
+<span class="sd"> :param cc: list of recipients to be added in CC field</span>
+<span class="sd"> :type cc: list or string (comma or semicolon delimited)</span>
+<span class="sd"> :param bcc: list of recipients to be added in BCC field</span>
+<span class="sd"> :type bcc: list or string (comma or semicolon delimited)</span>
+<span class="sd"> :param mime_subtype: MIME sub content type</span>
+<span class="sd"> :type mime_subtype: string</span>
+<span class="sd"> :param mime_charset: character set parameter added to the Content-Type</span>
+<span class="sd"> header.</span>
+<span class="sd"> :type mime_charset: string</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'to'</span><span class="p">,</span> <span class="s1">'subject'</span><span class="p">,</span> <span class="s1">'html_content'</span><span class="p">)</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.html'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#e6faf9'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">to</span><span class="p">,</span>
+ <span class="n">subject</span><span class="p">,</span>
+ <span class="n">html_content</span><span class="p">,</span>
+ <span class="n">files</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">cc</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">bcc</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">mime_subtype</span><span class="o">=</span><span class="s1">'mixed'</span><span class="p">,</span>
+ <span class="n">mime_charset</span><span class="o">=</span><span class="s1">'us_ascii'</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">EmailOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">to</span> <span class="o">=</span> <span class="n">to</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">subject</span> <span class="o">=</span> <span class="n">subject</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">html_content</span> <span class="o">=</span> <span class="n">html_content</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">files</span> <span class="o">=</span> <span class="n">files</span> <span class="ow">or</span> <span class="p">[]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cc</span> <span class="o">=</span> <span class="n">cc</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bcc</span> <span class="o">=</span> <span class="n">bcc</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mime_subtype</span> <span class="o">=</span> <span class="n">mime_subtype</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mime_charset</span> <span class="o">=</span> <span class="n">mime_charset</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">send_email</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">subject</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">html_content</span><span class="p">,</span>
+ <span class="n">files</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">files</span><span class="p">,</span> <span class="n">cc</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cc</span><span class="p">,</span> <span class="n">bcc</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bcc</span><span class="p">,</span>
+ <span class="n">mime_subtype</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mime_subtype</span><span class="p">,</span> <span class="n">mine_charset</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mime_charset</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/generic_transfer.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/generic_transfer.html b/_modules/airflow/operators/generic_transfer.html
new file mode 100644
index 0000000..4a03958
--- /dev/null
+++ b/_modules/airflow/operators/generic_transfer.html
@@ -0,0 +1,315 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.operators.generic_transfer — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.operators.generic_transfer</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.operators.generic_transfer</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+
+
+<div class="viewcode-block" id="GenericTransfer"><a class="viewcode-back" href="../../../code.html#airflow.operators.generic_transfer.GenericTransfer">[docs]</a><span class="k">class</span> <span class="nc">GenericTransfer</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Moves data from a connection to another, assuming that they both</span>
+<span class="sd"> provide the required methods in their respective hooks. The source hook</span>
+<span class="sd"> needs to expose a `get_records` method, and the destination a</span>
+<span class="sd"> `insert_rows` method.</span>
+
+<span class="sd"> This is meant to be used on small-ish datasets that fit in memory.</span>
+
+<span class="sd"> :param sql: SQL query to execute against the source database</span>
+<span class="sd"> :type sql: str</span>
+<span class="sd"> :param destination_table: target table</span>
+<span class="sd"> :type destination_table: str</span>
+<span class="sd"> :param source_conn_id: source connection</span>
+<span class="sd"> :type source_conn_id: str</span>
+<span class="sd"> :param destination_conn_id: source connection</span>
+<span class="sd"> :type destination_conn_id: str</span>
+<span class="sd"> :param preoperator: sql statement or list of statements to be</span>
+<span class="sd"> executed prior to loading the data</span>
+<span class="sd"> :type preoperator: str or list of str</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'sql'</span><span class="p">,</span> <span class="s1">'destination_table'</span><span class="p">,</span> <span class="s1">'preoperator'</span><span class="p">)</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,</span> <span class="s1">'.hql'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#b0f07c'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">sql</span><span class="p">,</span>
+ <span class="n">destination_table</span><span class="p">,</span>
+ <span class="n">source_conn_id</span><span class="p">,</span>
+ <span class="n">destination_conn_id</span><span class="p">,</span>
+ <span class="n">preoperator</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">GenericTransfer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">destination_table</span> <span class="o">=</span> <span class="n">destination_table</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">source_conn_id</span> <span class="o">=</span> <span class="n">source_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">destination_conn_id</span> <span class="o">=</span> <span class="n">destination_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">preoperator</span> <span class="o">=</span> <span class="n">preoperator</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">source_hook</span> <span class="o">=</span> <span class="n">BaseHook</span><span class="o">.</span><span class="n">get_hook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">source_conn_id</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Extracting data from </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">source_conn_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Executing: </span><span class="se">\n</span><span class="s2"> </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
+ <span class="n">results</span> <span class="o">=</span> <span class="n">source_hook</span><span class="o">.</span><span class="n">get_records</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
+
+ <span class="n">destination_hook</span> <span class="o">=</span> <span class="n">BaseHook</span><span class="o">.</span><span class="n">get_hook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">destination_conn_id</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">preoperator</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Running preoperator"</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">preoperator</span><span class="p">)</span>
+ <span class="n">destination_hook</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">preoperator</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Inserting rows into </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">destination_conn_id</span><span class="p">)</span>
+ <span class="n">destination_hook</span><span class="o">.</span><span class="n">insert_rows</span><span class="p">(</span><span class="n">table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">destination_table</span><span class="p">,</span> <span class="n">rows</span><span class="o">=</span><span class="n">results</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[10/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/http_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/http_operator.html b/_modules/airflow/operators/http_operator.html
new file mode 100644
index 0000000..5890dde
--- /dev/null
+++ b/_modules/airflow/operators/http_operator.html
@@ -0,0 +1,327 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.operators.http_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.operators.http_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.operators.http_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.http_hook</span> <span class="k">import</span> <span class="n">HttpHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="SimpleHttpOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.http_operator.SimpleHttpOperator">[docs]</a><span class="k">class</span> <span class="nc">SimpleHttpOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Calls an endpoint on an HTTP system to execute an action</span>
+
+<span class="sd"> :param http_conn_id: The connection to run the sensor against</span>
+<span class="sd"> :type http_conn_id: string</span>
+<span class="sd"> :param endpoint: The relative part of the full url</span>
+<span class="sd"> :type endpoint: string</span>
+<span class="sd"> :param method: The HTTP method to use, default = "POST"</span>
+<span class="sd"> :type method: string</span>
+<span class="sd"> :param data: The data to pass. POST-data in POST/PUT and params</span>
+<span class="sd"> in the URL for a GET request.</span>
+<span class="sd"> :type data: For POST/PUT, depends on the content-type parameter,</span>
+<span class="sd"> for GET a dictionary of key/value string pairs</span>
+<span class="sd"> :param headers: The HTTP headers to be added to the GET request</span>
+<span class="sd"> :type headers: a dictionary of string key/value pairs</span>
+<span class="sd"> :param response_check: A check against the 'requests' response object.</span>
+<span class="sd"> Returns True for 'pass' and False otherwise.</span>
+<span class="sd"> :type response_check: A lambda or defined function.</span>
+<span class="sd"> :param extra_options: Extra options for the 'requests' library, see the</span>
+<span class="sd"> 'requests' documentation (options to modify timeout, ssl, etc.)</span>
+<span class="sd"> :type extra_options: A dictionary of options, where key is string and value</span>
+<span class="sd"> depends on the option that's being modified.</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'endpoint'</span><span class="p">,</span> <span class="s1">'data'</span><span class="p">,)</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">()</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f4a460'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">endpoint</span><span class="p">,</span>
+ <span class="n">method</span><span class="o">=</span><span class="s1">'POST'</span><span class="p">,</span>
+ <span class="n">data</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">headers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">response_check</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">extra_options</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">xcom_push</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">http_conn_id</span><span class="o">=</span><span class="s1">'http_default'</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> If xcom_push is True, response of an HTTP request will also</span>
+<span class="sd"> be pushed to an XCom.</span>
+<span class="sd"> """</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">SimpleHttpOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">http_conn_id</span> <span class="o">=</span> <span class="n">http_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">=</span> <span class="n">method</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">endpoint</span> <span class="o">=</span> <span class="n">endpoint</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">headers</span> <span class="o">=</span> <span class="n">headers</span> <span class="ow">or</span> <span class="p">{}</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">data</span> <span class="o">=</span> <span class="n">data</span> <span class="ow">or</span> <span class="p">{}</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">response_check</span> <span class="o">=</span> <span class="n">response_check</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">extra_options</span> <span class="o">=</span> <span class="n">extra_options</span> <span class="ow">or</span> <span class="p">{}</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">xcom_push_flag</span> <span class="o">=</span> <span class="n">xcom_push</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">http</span> <span class="o">=</span> <span class="n">HttpHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">method</span><span class="p">,</span> <span class="n">http_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">http_conn_id</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Calling HTTP method"</span><span class="p">)</span>
+
+ <span class="n">response</span> <span class="o">=</span> <span class="n">http</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">endpoint</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">headers</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">extra_options</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">response_check</span><span class="p">:</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">response_check</span><span class="p">(</span><span class="n">response</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Response check returned False."</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">xcom_push_flag</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">response</span><span class="o">.</span><span class="n">text</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/latest_only_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/latest_only_operator.html b/_modules/airflow/operators/latest_only_operator.html
new file mode 100644
index 0000000..db14ae8
--- /dev/null
+++ b/_modules/airflow/operators/latest_only_operator.html
@@ -0,0 +1,300 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.operators.latest_only_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.operators.latest_only_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.operators.latest_only_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span><span class="p">,</span> <span class="n">SkipMixin</span>
+<span class="kn">from</span> <span class="nn">airflow.utils</span> <span class="k">import</span> <span class="n">timezone</span>
+
+
+<div class="viewcode-block" id="LatestOnlyOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.latest_only_operator.LatestOnlyOperator">[docs]</a><span class="k">class</span> <span class="nc">LatestOnlyOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">,</span> <span class="n">SkipMixin</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Allows a workflow to skip tasks that are not running during the most</span>
+<span class="sd"> recent schedule interval.</span>
+
+<span class="sd"> If the task is run outside of the latest schedule interval, all</span>
+<span class="sd"> directly downstream tasks will be skipped.</span>
+<span class="sd"> """</span>
+
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#e9ffdb'</span> <span class="c1"># nyanza</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="c1"># If the DAG Run is externally triggered, then return without</span>
+ <span class="c1"># skipping downstream tasks</span>
+ <span class="k">if</span> <span class="n">context</span><span class="p">[</span><span class="s1">'dag_run'</span><span class="p">]</span> <span class="ow">and</span> <span class="n">context</span><span class="p">[</span><span class="s1">'dag_run'</span><span class="p">]</span><span class="o">.</span><span class="n">external_trigger</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Externally triggered DAG_Run: allowing execution to proceed."</span><span class="p">)</span>
+ <span class="k">return</span>
+
+ <span class="n">now</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
+ <span class="n">left_window</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">'dag'</span><span class="p">]</span><span class="o">.</span><span class="n">following_schedule</span><span class="p">(</span>
+ <span class="n">context</span><span class="p">[</span><span class="s1">'execution_date'</span><span class="p">])</span>
+ <span class="n">right_window</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">'dag'</span><span class="p">]</span><span class="o">.</span><span class="n">following_schedule</span><span class="p">(</span><span class="n">left_window</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'Checking latest only with left_window: </span><span class="si">%s</span><span class="s1"> right_window: </span><span class="si">%s</span><span class="s1"> now: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">left_window</span><span class="p">,</span> <span class="n">right_window</span><span class="p">,</span> <span class="n">now</span>
+ <span class="p">)</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">left_window</span> <span class="o"><</span> <span class="n">now</span> <span class="o"><=</span> <span class="n">right_window</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Not latest execution, skipping downstream.'</span><span class="p">)</span>
+
+ <span class="n">downstream_tasks</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">'task'</span><span class="p">]</span><span class="o">.</span><span class="n">get_flat_relatives</span><span class="p">(</span><span class="n">upstream</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Downstream task_ids </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">downstream_tasks</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">downstream_tasks</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">skip</span><span class="p">(</span><span class="n">context</span><span class="p">[</span><span class="s1">'dag_run'</span><span class="p">],</span>
+ <span class="n">context</span><span class="p">[</span><span class="s1">'ti'</span><span class="p">]</span><span class="o">.</span><span class="n">execution_date</span><span class="p">,</span>
+ <span class="n">downstream_tasks</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Done.'</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Latest, allowing execution to proceed.'</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/mssql_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/mssql_operator.html b/_modules/airflow/operators/mssql_operator.html
new file mode 100644
index 0000000..38c1abd
--- /dev/null
+++ b/_modules/airflow/operators/mssql_operator.html
@@ -0,0 +1,291 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.operators.mssql_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.operators.mssql_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.operators.mssql_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.mssql_hook</span> <span class="k">import</span> <span class="n">MsSqlHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="MsSqlOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.mssql_operator.MsSqlOperator">[docs]</a><span class="k">class</span> <span class="nc">MsSqlOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Executes sql code in a specific Microsoft SQL database</span>
+
+<span class="sd"> :param mssql_conn_id: reference to a specific mssql database</span>
+<span class="sd"> :type mssql_conn_id: string</span>
+<span class="sd"> :param sql: the sql code to be executed</span>
+<span class="sd"> :type sql: string or string pointing to a template file with .sql extension</span>
+<span class="sd"> :param database: name of database which overwrite defined one in connection</span>
+<span class="sd"> :type database: string</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'sql'</span><span class="p">,)</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#ededed'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">mssql_conn_id</span><span class="o">=</span><span class="s1">'mssql_default'</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">autocommit</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">database</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">MsSqlOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mssql_conn_id</span> <span class="o">=</span> <span class="n">mssql_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span> <span class="o">=</span> <span class="n">parameters</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">autocommit</span> <span class="o">=</span> <span class="n">autocommit</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">database</span> <span class="o">=</span> <span class="n">database</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">MsSqlHook</span><span class="p">(</span><span class="n">mssql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mssql_conn_id</span><span class="p">,</span>
+ <span class="n">schema</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">database</span><span class="p">)</span>
+ <span class="n">hook</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">,</span> <span class="n">autocommit</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">autocommit</span><span class="p">,</span>
+ <span class="n">parameters</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">parameters</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/mysql_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/mysql_operator.html b/_modules/airflow/operators/mysql_operator.html
new file mode 100644
index 0000000..c5dd03a
--- /dev/null
+++ b/_modules/airflow/operators/mysql_operator.html
@@ -0,0 +1,295 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.operators.mysql_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.operators.mysql_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.operators.mysql_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="k">import</span> <span class="n">MySqlHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="MySqlOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.mysql_operator.MySqlOperator">[docs]</a><span class="k">class</span> <span class="nc">MySqlOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Executes sql code in a specific MySQL database</span>
+
+<span class="sd"> :param mysql_conn_id: reference to a specific mysql database</span>
+<span class="sd"> :type mysql_conn_id: string</span>
+<span class="sd"> :param sql: the sql code to be executed</span>
+<span class="sd"> :type sql: Can receive a str representing a sql statement,</span>
+<span class="sd"> a list of str (sql statements), or reference to a template file.</span>
+<span class="sd"> Template reference are recognized by str ending in '.sql'</span>
+<span class="sd"> :param database: name of database which overwrite defined one in connection</span>
+<span class="sd"> :type database: string</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'sql'</span><span class="p">,)</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#ededed'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">mysql_conn_id</span><span class="o">=</span><span class="s1">'mysql_default'</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">autocommit</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">database</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">MySqlOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mysql_conn_id</span> <span class="o">=</span> <span class="n">mysql_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">autocommit</span> <span class="o">=</span> <span class="n">autocommit</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span> <span class="o">=</span> <span class="n">parameters</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">database</span> <span class="o">=</span> <span class="n">database</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">MySqlHook</span><span class="p">(</span><span class="n">mysql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mysql_conn_id</span><span class="p">,</span>
+ <span class="n">schema</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">database</span><span class="p">)</span>
+ <span class="n">hook</span><span class="o">.</span><span class="n">run</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">,</span>
+ <span class="n">autocommit</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">autocommit</span><span class="p">,</span>
+ <span class="n">parameters</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">parameters</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/oracle_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/oracle_operator.html b/_modules/airflow/operators/oracle_operator.html
new file mode 100644
index 0000000..c8e8baf
--- /dev/null
+++ b/_modules/airflow/operators/oracle_operator.html
@@ -0,0 +1,290 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.operators.oracle_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.operators.oracle_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.operators.oracle_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.oracle_hook</span> <span class="k">import</span> <span class="n">OracleHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="OracleOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.oracle_operator.OracleOperator">[docs]</a><span class="k">class</span> <span class="nc">OracleOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Executes sql code in a specific Oracle database</span>
+<span class="sd"> :param oracle_conn_id: reference to a specific Oracle database</span>
+<span class="sd"> :type oracle_conn_id: string</span>
+<span class="sd"> :param sql: the sql code to be executed</span>
+<span class="sd"> :type sql: Can receive a str representing a sql statement,</span>
+<span class="sd"> a list of str (sql statements), or reference to a template file.</span>
+<span class="sd"> Template reference are recognized by str ending in '.sql'</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'sql'</span><span class="p">,)</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#ededed'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">oracle_conn_id</span><span class="o">=</span><span class="s1">'oracle_default'</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">autocommit</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">OracleOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">oracle_conn_id</span> <span class="o">=</span> <span class="n">oracle_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">autocommit</span> <span class="o">=</span> <span class="n">autocommit</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span> <span class="o">=</span> <span class="n">parameters</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">OracleHook</span><span class="p">(</span><span class="n">oracle_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">oracle_conn_id</span><span class="p">)</span>
+ <span class="n">hook</span><span class="o">.</span><span class="n">run</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">,</span>
+ <span class="n">autocommit</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">autocommit</span><span class="p">,</span>
+ <span class="n">parameters</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">parameters</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/pig_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/pig_operator.html b/_modules/airflow/operators/pig_operator.html
new file mode 100644
index 0000000..0331036
--- /dev/null
+++ b/_modules/airflow/operators/pig_operator.html
@@ -0,0 +1,307 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.operators.pig_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.operators.pig_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.operators.pig_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">import</span> <span class="nn">re</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.pig_hook</span> <span class="k">import</span> <span class="n">PigCliHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="PigOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.pig_operator.PigOperator">[docs]</a><span class="k">class</span> <span class="nc">PigOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Executes pig script.</span>
+
+<span class="sd"> :param pig: the pig latin script to be executed</span>
+<span class="sd"> :type pig: string</span>
+<span class="sd"> :param pig_cli_conn_id: reference to the Hive database</span>
+<span class="sd"> :type pig_cli_conn_id: string</span>
+<span class="sd"> :param pigparams_jinja_translate: when True, pig params-type templating</span>
+<span class="sd"> ${var} gets translated into jinja-type templating {{ var }}. Note that</span>
+<span class="sd"> you may want to use this along with the</span>
+<span class="sd"> ``DAG(user_defined_macros=myargs)`` parameter. View the DAG</span>
+<span class="sd"> object documentation for more details.</span>
+<span class="sd"> :type pigparams_jinja_translate: boolean</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'pig'</span><span class="p">,)</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.pig'</span><span class="p">,</span> <span class="s1">'.piglatin'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f0e4ec'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span> <span class="n">pig</span><span class="p">,</span>
+ <span class="n">pig_cli_conn_id</span><span class="o">=</span><span class="s1">'pig_cli_default'</span><span class="p">,</span>
+ <span class="n">pigparams_jinja_translate</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+
+ <span class="nb">super</span><span class="p">(</span><span class="n">PigOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">pigparams_jinja_translate</span> <span class="o">=</span> <span class="n">pigparams_jinja_translate</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">pig</span> <span class="o">=</span> <span class="n">pig</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">pig_cli_conn_id</span> <span class="o">=</span> <span class="n">pig_cli_conn_id</span>
+
+ <span class="k">def</span> <span class="nf">get_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">PigCliHook</span><span class="p">(</span><span class="n">pig_cli_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">pig_cli_conn_id</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">prepare_template</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">pigparams_jinja_translate</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">pig</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span>
+ <span class="s2">"(\$([a-zA-Z_][a-zA-Z0-9_]*))"</span><span class="p">,</span> <span class="s2">"{{ \g<2> }}"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">pig</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">pig</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">run_cli</span><span class="p">(</span><span class="n">pig</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">pig</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">kill</span><span class="p">()</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[06/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/operators/sensors.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/sensors.html b/_modules/airflow/operators/sensors.html
deleted file mode 100644
index 7414daf..0000000
--- a/_modules/airflow/operators/sensors.html
+++ /dev/null
@@ -1,929 +0,0 @@
-
-
-<!DOCTYPE html>
-<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
-<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
-<head>
- <meta charset="utf-8">
-
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
-
- <title>airflow.operators.sensors — Airflow Documentation</title>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
- <link rel="up" title="Module code" href="../../index.html"/>
-
-
- <script src="../../../_static/js/modernizr.min.js"></script>
-
-</head>
-
-<body class="wy-body-for-nav" role="document">
-
-
- <div class="wy-grid-for-nav">
-
-
- <nav data-toggle="wy-nav-shift" class="wy-nav-side">
- <div class="wy-side-scroll">
- <div class="wy-side-nav-search">
-
-
-
- <a href="../../../index.html" class="icon icon-home"> Airflow
-
-
-
- </a>
-
-
-
-
-
-
-
-<div role="search">
- <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
- <input type="text" name="q" placeholder="Search docs" />
- <input type="hidden" name="check_keywords" value="yes" />
- <input type="hidden" name="area" value="default" />
- </form>
-</div>
-
-
- </div>
-
- <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
-
-
-
-
-
-
- <ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../configuration.html">Configuration</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
-</ul>
-
-
-
- </div>
- </div>
- </nav>
-
- <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
-
-
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
-
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="../../../index.html">Airflow</a>
-
- </nav>
-
-
-
- <div class="wy-nav-content">
- <div class="rst-content">
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-<div role="navigation" aria-label="breadcrumbs navigation">
-
- <ul class="wy-breadcrumbs">
-
- <li><a href="../../../index.html">Docs</a> »</li>
-
- <li><a href="../../index.html">Module code</a> »</li>
-
- <li>airflow.operators.sensors</li>
-
-
- <li class="wy-breadcrumbs-aside">
-
-
-
- </li>
-
- </ul>
-
-
- <hr/>
-</div>
- <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
- <div itemprop="articleBody">
-
- <h1>Source code for airflow.operators.sensors</h1><div class="highlight"><pre>
-<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
-<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
-
-<span class="kn">from</span> <span class="nn">__future__</span> <span class="k">import</span> <span class="n">print_function</span>
-<span class="kn">from</span> <span class="nn">future</span> <span class="k">import</span> <span class="n">standard_library</span>
-
-<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
-
-<span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
-<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
-<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="k">import</span> <span class="n">basestring</span>
-
-<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span>
-<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="k">import</span> <span class="n">urlparse</span>
-<span class="kn">from</span> <span class="nn">time</span> <span class="k">import</span> <span class="n">sleep</span>
-<span class="kn">import</span> <span class="nn">re</span>
-<span class="kn">import</span> <span class="nn">sys</span>
-
-<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">settings</span>
-<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span><span class="p">,</span> <span class="n">AirflowSensorTimeout</span><span class="p">,</span> <span class="n">AirflowSkipException</span>
-<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span><span class="p">,</span> <span class="n">TaskInstance</span>
-<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
-<span class="kn">from</span> <span class="nn">airflow.hooks.hdfs_hook</span> <span class="k">import</span> <span class="n">HDFSHook</span>
-<span class="kn">from</span> <span class="nn">airflow.hooks.http_hook</span> <span class="k">import</span> <span class="n">HttpHook</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
-
-
-<div class="viewcode-block" id="BaseSensorOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.sensors.BaseSensorOperator">[docs]</a><span class="k">class</span> <span class="nc">BaseSensorOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
- <span class="sd">'''</span>
-<span class="sd"> Sensor operators are derived from this class an inherit these attributes.</span>
-
-<span class="sd"> Sensor operators keep executing at a time interval and succeed when</span>
-<span class="sd"> a criteria is met and fail if and when they time out.</span>
-
-<span class="sd"> :param soft_fail: Set to true to mark the task as SKIPPED on failure</span>
-<span class="sd"> :type soft_fail: bool</span>
-<span class="sd"> :param poke_interval: Time in seconds that the job should wait in</span>
-<span class="sd"> between each tries</span>
-<span class="sd"> :type poke_interval: int</span>
-<span class="sd"> :param timeout: Time, in seconds before the task times out and fails.</span>
-<span class="sd"> :type timeout: int</span>
-<span class="sd"> '''</span>
- <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#e6f1f2'</span>
-
- <span class="nd">@apply_defaults</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
- <span class="bp">self</span><span class="p">,</span>
- <span class="n">poke_interval</span><span class="o">=</span><span class="mi">60</span><span class="p">,</span>
- <span class="n">timeout</span><span class="o">=</span><span class="mi">60</span><span class="o">*</span><span class="mi">60</span><span class="o">*</span><span class="mi">24</span><span class="o">*</span><span class="mi">7</span><span class="p">,</span>
- <span class="n">soft_fail</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">poke_interval</span> <span class="o">=</span> <span class="n">poke_interval</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">soft_fail</span> <span class="o">=</span> <span class="n">soft_fail</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span> <span class="o">=</span> <span class="n">timeout</span>
-
- <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
- <span class="sd">'''</span>
-<span class="sd"> Function that the sensors defined while deriving this class should</span>
-<span class="sd"> override.</span>
-<span class="sd"> '''</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Override me.'</span><span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
- <span class="n">started_at</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
- <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">poke</span><span class="p">(</span><span class="n">context</span><span class="p">):</span>
- <span class="k">if</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">-</span> <span class="n">started_at</span><span class="p">)</span><span class="o">.</span><span class="n">total_seconds</span><span class="p">()</span> <span class="o">></span> <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">:</span>
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">soft_fail</span><span class="p">:</span>
- <span class="k">raise</span> <span class="n">AirflowSkipException</span><span class="p">(</span><span class="s1">'Snap. Time is OUT.'</span><span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="k">raise</span> <span class="n">AirflowSensorTimeout</span><span class="p">(</span><span class="s1">'Snap. Time is OUT.'</span><span class="p">)</span>
- <span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">poke_interval</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Success criteria met. Exiting."</span><span class="p">)</span></div>
-
-
-<span class="k">class</span> <span class="nc">SqlSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Runs a sql statement until a criteria is met. It will keep trying while</span>
-<span class="sd"> sql returns no row, or if the first cell in (0, '0', '').</span>
-
-<span class="sd"> :param conn_id: The connection to run the sensor against</span>
-<span class="sd"> :type conn_id: string</span>
-<span class="sd"> :param sql: The sql to run. To pass, it needs to return at least one cell</span>
-<span class="sd"> that contains a non-zero / empty string value.</span>
-<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'sql'</span><span class="p">,)</span>
- <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.hql'</span><span class="p">,</span> <span class="s1">'.sql'</span><span class="p">,)</span>
- <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#7c7287'</span>
-
- <span class="nd">@apply_defaults</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn_id</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span> <span class="o">=</span> <span class="n">conn_id</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">SqlSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
- <span class="n">hook</span> <span class="o">=</span> <span class="n">BaseHook</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span>
-
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Poking: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
- <span class="n">records</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_records</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">records</span><span class="p">:</span>
- <span class="k">return</span> <span class="kc">False</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="k">if</span> <span class="nb">str</span><span class="p">(</span><span class="n">records</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="mi">0</span><span class="p">])</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'0'</span><span class="p">,</span> <span class="s1">''</span><span class="p">,):</span>
- <span class="k">return</span> <span class="kc">False</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="k">return</span> <span class="kc">True</span>
-
-
-<span class="k">class</span> <span class="nc">MetastorePartitionSensor</span><span class="p">(</span><span class="n">SqlSensor</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> An alternative to the HivePartitionSensor that talk directly to the</span>
-<span class="sd"> MySQL db. This was created as a result of observing sub optimal</span>
-<span class="sd"> queries generated by the Metastore thrift service when hitting</span>
-<span class="sd"> subpartitioned tables. The Thrift service's queries were written in a</span>
-<span class="sd"> way that wouldn't leverage the indexes.</span>
-
-<span class="sd"> :param schema: the schema</span>
-<span class="sd"> :type schema: str</span>
-<span class="sd"> :param table: the table</span>
-<span class="sd"> :type table: str</span>
-<span class="sd"> :param partition_name: the partition name, as defined in the PARTITIONS</span>
-<span class="sd"> table of the Metastore. Order of the fields does matter.</span>
-<span class="sd"> Examples: ``ds=2016-01-01`` or</span>
-<span class="sd"> ``ds=2016-01-01/sub=foo`` for a sub partitioned table</span>
-<span class="sd"> :type partition_name: str</span>
-<span class="sd"> :param mysql_conn_id: a reference to the MySQL conn_id for the metastore</span>
-<span class="sd"> :type mysql_conn_id: str</span>
-<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'partition_name'</span><span class="p">,</span> <span class="s1">'table'</span><span class="p">,</span> <span class="s1">'schema'</span><span class="p">)</span>
- <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#8da7be'</span>
-
- <span class="nd">@apply_defaults</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
- <span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition_name</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="s2">"default"</span><span class="p">,</span>
- <span class="n">mysql_conn_id</span><span class="o">=</span><span class="s2">"metastore_mysql"</span><span class="p">,</span>
- <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
-
- <span class="bp">self</span><span class="o">.</span><span class="n">partition_name</span> <span class="o">=</span> <span class="n">partition_name</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">table</span> <span class="o">=</span> <span class="n">table</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="o">=</span> <span class="n">schema</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">first_poke</span> <span class="o">=</span> <span class="kc">True</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span> <span class="o">=</span> <span class="n">mysql_conn_id</span>
- <span class="c1"># TODO(aoen): We shouldn't be using SqlSensor here but MetastorePartitionSensor.</span>
- <span class="c1"># The problem is the way apply_defaults works isn't compatible with inheritance.</span>
- <span class="c1"># The inheritance model needs to be reworked in order to support overriding args/</span>
- <span class="c1"># kwargs with arguments here, then 'conn_id' and 'sql' can be passed into the</span>
- <span class="c1"># constructor below and apply_defaults will no longer throw an exception.</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">SqlSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">first_poke</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">first_poke</span> <span class="o">=</span> <span class="kc">False</span>
- <span class="k">if</span> <span class="s1">'.'</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="s2">"""</span>
-<span class="s2"> SELECT 'X'</span>
-<span class="s2"> FROM PARTITIONS A0</span>
-<span class="s2"> LEFT OUTER JOIN TBLS B0 ON A0.TBL_ID = B0.TBL_ID</span>
-<span class="s2"> LEFT OUTER JOIN DBS C0 ON B0.DB_ID = C0.DB_ID</span>
-<span class="s2"> WHERE</span>
-<span class="s2"> B0.TBL_NAME = '</span><span class="si">{self.table}</span><span class="s2">' AND</span>
-<span class="s2"> C0.NAME = '</span><span class="si">{self.schema}</span><span class="s2">' AND</span>
-<span class="s2"> A0.PART_NAME = '</span><span class="si">{self.partition_name}</span><span class="s2">';</span>
-<span class="s2"> """</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">=</span><span class="bp">self</span><span class="p">)</span>
- <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">MetastorePartitionSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">poke</span><span class="p">(</span><span class="n">context</span><span class="p">)</span>
-
-
-<span class="k">class</span> <span class="nc">ExternalTaskSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Waits for a task to complete in a different DAG</span>
-
-<span class="sd"> :param external_dag_id: The dag_id that contains the task you want to</span>
-<span class="sd"> wait for</span>
-<span class="sd"> :type external_dag_id: string</span>
-<span class="sd"> :param external_task_id: The task_id that contains the task you want to</span>
-<span class="sd"> wait for</span>
-<span class="sd"> :type external_task_id: string</span>
-<span class="sd"> :param allowed_states: list of allowed states, default is ``['success']``</span>
-<span class="sd"> :type allowed_states: list</span>
-<span class="sd"> :param execution_delta: time difference with the previous execution to</span>
-<span class="sd"> look at, the default is the same execution_date as the current task.</span>
-<span class="sd"> For yesterday, use [positive!] datetime.timedelta(days=1). Either</span>
-<span class="sd"> execution_delta or execution_date_fn can be passed to</span>
-<span class="sd"> ExternalTaskSensor, but not both.</span>
-<span class="sd"> :type execution_delta: datetime.timedelta</span>
-<span class="sd"> :param execution_date_fn: function that receives the current execution date</span>
-<span class="sd"> and returns the desired execution dates to query. Either execution_delta</span>
-<span class="sd"> or execution_date_fn can be passed to ExternalTaskSensor, but not both.</span>
-<span class="sd"> :type execution_date_fn: callable</span>
-<span class="sd"> """</span>
- <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#19647e'</span>
-
- <span class="nd">@apply_defaults</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
- <span class="bp">self</span><span class="p">,</span>
- <span class="n">external_dag_id</span><span class="p">,</span>
- <span class="n">external_task_id</span><span class="p">,</span>
- <span class="n">allowed_states</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">execution_delta</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">execution_date_fn</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">ExternalTaskSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">allowed_states</span> <span class="o">=</span> <span class="n">allowed_states</span> <span class="ow">or</span> <span class="p">[</span><span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">]</span>
- <span class="k">if</span> <span class="n">execution_delta</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">execution_date_fn</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
- <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
- <span class="s1">'Only one of `execution_date` or `execution_date_fn` may'</span>
- <span class="s1">'be provided to ExternalTaskSensor; not both.'</span><span class="p">)</span>
-
- <span class="bp">self</span><span class="o">.</span><span class="n">execution_delta</span> <span class="o">=</span> <span class="n">execution_delta</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">execution_date_fn</span> <span class="o">=</span> <span class="n">execution_date_fn</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">external_dag_id</span> <span class="o">=</span> <span class="n">external_dag_id</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">external_task_id</span> <span class="o">=</span> <span class="n">external_task_id</span>
-
- <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_delta</span><span class="p">:</span>
- <span class="n">dttm</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">'execution_date'</span><span class="p">]</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_delta</span>
- <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date_fn</span><span class="p">:</span>
- <span class="n">dttm</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date_fn</span><span class="p">(</span><span class="n">context</span><span class="p">[</span><span class="s1">'execution_date'</span><span class="p">])</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">dttm</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">'execution_date'</span><span class="p">]</span>
-
- <span class="n">dttm_filter</span> <span class="o">=</span> <span class="n">dttm</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dttm</span><span class="p">,</span> <span class="nb">list</span><span class="p">)</span> <span class="k">else</span> <span class="p">[</span><span class="n">dttm</span><span class="p">]</span>
- <span class="n">serialized_dttm_filter</span> <span class="o">=</span> <span class="s1">','</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
- <span class="p">[</span><span class="n">datetime</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span> <span class="k">for</span> <span class="n">datetime</span> <span class="ow">in</span> <span class="n">dttm_filter</span><span class="p">])</span>
-
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
- <span class="s1">'Poking for '</span>
- <span class="s1">'</span><span class="si">{self.external_dag_id}</span><span class="s1">.'</span>
- <span class="s1">'</span><span class="si">{self.external_task_id}</span><span class="s1"> on '</span>
- <span class="s1">'</span><span class="si">{}</span><span class="s1"> ... '</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">serialized_dttm_filter</span><span class="p">,</span> <span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
- <span class="n">TI</span> <span class="o">=</span> <span class="n">TaskInstance</span>
-
- <span class="n">session</span> <span class="o">=</span> <span class="n">settings</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span>
- <span class="n">count</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TI</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
- <span class="n">TI</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">external_dag_id</span><span class="p">,</span>
- <span class="n">TI</span><span class="o">.</span><span class="n">task_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">external_task_id</span><span class="p">,</span>
- <span class="n">TI</span><span class="o">.</span><span class="n">state</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">allowed_states</span><span class="p">),</span>
- <span class="n">TI</span><span class="o">.</span><span class="n">execution_date</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">dttm_filter</span><span class="p">),</span>
- <span class="p">)</span><span class="o">.</span><span class="n">count</span><span class="p">()</span>
- <span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
- <span class="n">session</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
- <span class="k">return</span> <span class="n">count</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">dttm_filter</span><span class="p">)</span>
-
-
-<span class="k">class</span> <span class="nc">NamedHivePartitionSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Waits for a set of partitions to show up in Hive.</span>
-
-<span class="sd"> :param partition_names: List of fully qualified names of the</span>
-<span class="sd"> partitions to wait for. A fully qualified name is of the</span>
-<span class="sd"> form ``schema.table/pk1=pv1/pk2=pv2``, for example,</span>
-<span class="sd"> default.users/ds=2016-01-01. This is passed as is to the metastore</span>
-<span class="sd"> Thrift client ``get_partitions_by_name`` method. Note that</span>
-<span class="sd"> you cannot use logical or comparison operators as in</span>
-<span class="sd"> HivePartitionSensor.</span>
-<span class="sd"> :type partition_names: list of strings</span>
-<span class="sd"> :param metastore_conn_id: reference to the metastore thrift service</span>
-<span class="sd"> connection id</span>
-<span class="sd"> :type metastore_conn_id: str</span>
-<span class="sd"> """</span>
-
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'partition_names'</span><span class="p">,</span> <span class="p">)</span>
- <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#8d99ae'</span>
-
- <span class="nd">@apply_defaults</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
- <span class="bp">self</span><span class="p">,</span>
- <span class="n">partition_names</span><span class="p">,</span>
- <span class="n">metastore_conn_id</span><span class="o">=</span><span class="s1">'metastore_default'</span><span class="p">,</span>
- <span class="n">poke_interval</span><span class="o">=</span><span class="mi">60</span> <span class="o">*</span> <span class="mi">3</span><span class="p">,</span>
- <span class="o">*</span><span class="n">args</span><span class="p">,</span>
- <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">NamedHivePartitionSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
- <span class="n">poke_interval</span><span class="o">=</span><span class="n">poke_interval</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
-
- <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">partition_names</span><span class="p">,</span> <span class="n">basestring</span><span class="p">):</span>
- <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">'partition_names must be an array of strings'</span><span class="p">)</span>
-
- <span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span> <span class="o">=</span> <span class="n">metastore_conn_id</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span> <span class="o">=</span> <span class="n">partition_names</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o">=</span> <span class="mi">0</span>
-
- <span class="nd">@classmethod</span>
- <span class="k">def</span> <span class="nf">parse_partition_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">partition</span><span class="p">):</span>
- <span class="k">try</span><span class="p">:</span>
- <span class="n">schema</span><span class="p">,</span> <span class="n">table_partition</span> <span class="o">=</span> <span class="n">partition</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
- <span class="n">table</span><span class="p">,</span> <span class="n">partition</span> <span class="o">=</span> <span class="n">table_partition</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'/'</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span>
- <span class="k">except</span> <span class="ne">ValueError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
- <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'Could not parse '</span> <span class="o">+</span> <span class="n">partition</span><span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">'hook'</span><span class="p">):</span>
- <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="k">import</span> <span class="n">HiveMetastoreHook</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">(</span>
- <span class="n">metastore_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span><span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">poke_partition</span><span class="p">(</span><span class="n">partition</span><span class="p">):</span>
-
- <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_partition_name</span><span class="p">(</span><span class="n">partition</span><span class="p">)</span>
-
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
- <span class="s1">'Poking for </span><span class="si">{schema}</span><span class="s1">.</span><span class="si">{table}</span><span class="s1">/</span><span class="si">{partition}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
- <span class="p">)</span>
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">check_for_named_partition</span><span class="p">(</span>
- <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span><span class="p">)</span>
-
- <span class="k">while</span> <span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o"><</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span><span class="p">):</span>
- <span class="k">if</span> <span class="n">poke_partition</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span><span class="p">]):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o">+=</span> <span class="mi">1</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="k">return</span> <span class="kc">False</span>
-
- <span class="k">return</span> <span class="kc">True</span>
-
-
-<span class="k">class</span> <span class="nc">HivePartitionSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Waits for a partition to show up in Hive.</span>
-
-<span class="sd"> Note: Because ``partition`` supports general logical operators, it</span>
-<span class="sd"> can be inefficient. Consider using NamedHivePartitionSensor instead if</span>
-<span class="sd"> you don't need the full flexibility of HivePartitionSensor.</span>
-
-<span class="sd"> :param table: The name of the table to wait for, supports the dot</span>
-<span class="sd"> notation (my_database.my_table)</span>
-<span class="sd"> :type table: string</span>
-<span class="sd"> :param partition: The partition clause to wait for. This is passed as</span>
-<span class="sd"> is to the metastore Thrift client ``get_partitions_by_filter`` method,</span>
-<span class="sd"> and apparently supports SQL like notation as in ``ds='2015-01-01'</span>
-<span class="sd"> AND type='value'`` and comparison operators as in ``"ds>=2015-01-01"``</span>
-<span class="sd"> :type partition: string</span>
-<span class="sd"> :param metastore_conn_id: reference to the metastore thrift service</span>
-<span class="sd"> connection id</span>
-<span class="sd"> :type metastore_conn_id: str</span>
-<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'schema'</span><span class="p">,</span> <span class="s1">'table'</span><span class="p">,</span> <span class="s1">'partition'</span><span class="p">,)</span>
- <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#C5CAE9'</span>
-
- <span class="nd">@apply_defaults</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
- <span class="bp">self</span><span class="p">,</span>
- <span class="n">table</span><span class="p">,</span> <span class="n">partition</span><span class="o">=</span><span class="s2">"ds='{{ ds }}'"</span><span class="p">,</span>
- <span class="n">metastore_conn_id</span><span class="o">=</span><span class="s1">'metastore_default'</span><span class="p">,</span>
- <span class="n">schema</span><span class="o">=</span><span class="s1">'default'</span><span class="p">,</span>
- <span class="n">poke_interval</span><span class="o">=</span><span class="mi">60</span><span class="o">*</span><span class="mi">3</span><span class="p">,</span>
- <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HivePartitionSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
- <span class="n">poke_interval</span><span class="o">=</span><span class="n">poke_interval</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">partition</span><span class="p">:</span>
- <span class="n">partition</span> <span class="o">=</span> <span class="s2">"ds='{{ ds }}'"</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span> <span class="o">=</span> <span class="n">metastore_conn_id</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">table</span> <span class="o">=</span> <span class="n">table</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">partition</span> <span class="o">=</span> <span class="n">partition</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="o">=</span> <span class="n">schema</span>
-
- <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
- <span class="k">if</span> <span class="s1">'.'</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
- <span class="s1">'Poking for table </span><span class="si">{self.schema}</span><span class="s1">.</span><span class="si">{self.table}</span><span class="s1">, '</span>
- <span class="s1">'partition </span><span class="si">{self.partition}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">'hook'</span><span class="p">):</span>
- <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="k">import</span> <span class="n">HiveMetastoreHook</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">(</span>
- <span class="n">metastore_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span><span class="p">)</span>
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">check_for_partition</span><span class="p">(</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">partition</span><span class="p">)</span>
-
-
-<span class="k">class</span> <span class="nc">HdfsSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Waits for a file or folder to land in HDFS</span>
-<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'filepath'</span><span class="p">,)</span>
- <span class="n">ui_color</span> <span class="o">=</span> <span class="n">settings</span><span class="o">.</span><span class="n">WEB_COLORS</span><span class="p">[</span><span class="s1">'LIGHTBLUE'</span><span class="p">]</span>
-
- <span class="nd">@apply_defaults</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
- <span class="bp">self</span><span class="p">,</span>
- <span class="n">filepath</span><span class="p">,</span>
- <span class="n">hdfs_conn_id</span><span class="o">=</span><span class="s1">'hdfs_default'</span><span class="p">,</span>
- <span class="n">ignored_ext</span><span class="o">=</span><span class="p">[</span><span class="s1">'_COPYING_'</span><span class="p">],</span>
- <span class="n">ignore_copying</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
- <span class="n">file_size</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">hook</span><span class="o">=</span><span class="n">HDFSHook</span><span class="p">,</span>
- <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HdfsSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">filepath</span> <span class="o">=</span> <span class="n">filepath</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span> <span class="o">=</span> <span class="n">hdfs_conn_id</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">file_size</span> <span class="o">=</span> <span class="n">file_size</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">ignored_ext</span> <span class="o">=</span> <span class="n">ignored_ext</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">ignore_copying</span> <span class="o">=</span> <span class="n">ignore_copying</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">hook</span>
-
- <span class="nd">@staticmethod</span>
- <span class="k">def</span> <span class="nf">filter_for_filesize</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">size</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Will test the filepath result and test if its size is at least self.filesize</span>
-
-<span class="sd"> :param result: a list of dicts returned by Snakebite ls</span>
-<span class="sd"> :param size: the file size in MB a file should be at least to trigger True</span>
-<span class="sd"> :return: (bool) depending on the matching criteria</span>
-<span class="sd"> """</span>
- <span class="k">if</span> <span class="n">size</span><span class="p">:</span>
- <span class="n">log</span> <span class="o">=</span> <span class="n">LoggingMixin</span><span class="p">()</span><span class="o">.</span><span class="n">log</span>
- <span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'Filtering for file size >= </span><span class="si">%s</span><span class="s1"> in files: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">size</span><span class="p">,</span> <span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="s1">'path'</span><span class="p">],</span> <span class="n">result</span><span class="p">))</span>
- <span class="n">size</span> <span class="o">*=</span> <span class="n">settings</span><span class="o">.</span><span class="n">MEGABYTE</span>
- <span class="n">result</span> <span class="o">=</span> <span class="p">[</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">result</span> <span class="k">if</span> <span class="n">x</span><span class="p">[</span><span class="s1">'length'</span><span class="p">]</span> <span class="o">>=</span> <span class="n">size</span><span class="p">]</span>
- <span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'HdfsSensor.poke: after size filter result is </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">result</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">result</span>
-
- <span class="nd">@staticmethod</span>
- <span class="k">def</span> <span class="nf">filter_for_ignored_ext</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">ignored_ext</span><span class="p">,</span> <span class="n">ignore_copying</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Will filter if instructed to do so the result to remove matching criteria</span>
-
-<span class="sd"> :param result: (list) of dicts returned by Snakebite ls</span>
-<span class="sd"> :param ignored_ext: (list) of ignored extensions</span>
-<span class="sd"> :param ignore_copying: (bool) shall we ignore ?</span>
-<span class="sd"> :return: (list) of dicts which were not removed</span>
-<span class="sd"> """</span>
- <span class="k">if</span> <span class="n">ignore_copying</span><span class="p">:</span>
- <span class="n">log</span> <span class="o">=</span> <span class="n">LoggingMixin</span><span class="p">()</span><span class="o">.</span><span class="n">log</span>
- <span class="n">regex_builder</span> <span class="o">=</span> <span class="s2">"^.*\.(</span><span class="si">%s</span><span class="s2">$)$"</span> <span class="o">%</span> <span class="s1">'$|'</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">ignored_ext</span><span class="p">)</span>
- <span class="n">ignored_extentions_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="n">regex_builder</span><span class="p">)</span>
- <span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
- <span class="s1">'Filtering result for ignored extensions: </span><span class="si">%s</span><span class="s1"> in files </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
- <span class="n">ignored_extentions_regex</span><span class="o">.</span><span class="n">pattern</span><span class="p">,</span> <span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="s1">'path'</span><span class="p">],</span> <span class="n">result</span><span class="p">)</span>
- <span class="p">)</span>
- <span class="n">result</span> <span class="o">=</span> <span class="p">[</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">result</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">ignored_extentions_regex</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="s1">'path'</span><span class="p">])]</span>
- <span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'HdfsSensor.poke: after ext filter result is </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">result</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">result</span>
-
- <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
- <span class="n">sb</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Poking for file </span><span class="si">{self.filepath}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
- <span class="k">try</span><span class="p">:</span>
- <span class="c1"># IMOO it's not right here, as there no raise of any kind.</span>
- <span class="c1"># if the filepath is let's say '/data/mydirectory', it's correct but if it is '/data/mydirectory/*',</span>
- <span class="c1"># it's not correct as the directory exists and sb does not raise any error</span>
- <span class="c1"># here is a quick fix</span>
- <span class="n">result</span> <span class="o">=</span> <span class="p">[</span><span class="n">f</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">sb</span><span class="o">.</span><span class="n">ls</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="p">],</span> <span class="n">include_toplevel</span><span class="o">=</span><span class="kc">False</span><span class="p">)]</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'HdfsSensor.poke: result is </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">result</span><span class="p">)</span>
- <span class="n">result</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">filter_for_ignored_ext</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ignored_ext</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ignore_copying</span><span class="p">)</span>
- <span class="n">result</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">filter_for_filesize</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">file_size</span><span class="p">)</span>
- <span class="k">return</span> <span class="nb">bool</span><span class="p">(</span><span class="n">result</span><span class="p">)</span>
- <span class="k">except</span><span class="p">:</span>
- <span class="n">e</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">exc_info</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Caught an exception !: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
- <span class="k">return</span> <span class="kc">False</span>
-
-
-<span class="k">class</span> <span class="nc">WebHdfsSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Waits for a file or folder to land in HDFS</span>
-<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'filepath'</span><span class="p">,)</span>
-
- <span class="nd">@apply_defaults</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
- <span class="bp">self</span><span class="p">,</span>
- <span class="n">filepath</span><span class="p">,</span>
- <span class="n">webhdfs_conn_id</span><span class="o">=</span><span class="s1">'webhdfs_default'</span><span class="p">,</span>
- <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">WebHdfsSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">filepath</span> <span class="o">=</span> <span class="n">filepath</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">webhdfs_conn_id</span> <span class="o">=</span> <span class="n">webhdfs_conn_id</span>
-
- <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
- <span class="kn">from</span> <span class="nn">airflow.hooks.webhdfs_hook</span> <span class="k">import</span> <span class="n">WebHDFSHook</span>
- <span class="n">c</span> <span class="o">=</span> <span class="n">WebHDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">webhdfs_conn_id</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Poking for file </span><span class="si">{self.filepath}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
- <span class="k">return</span> <span class="n">c</span><span class="o">.</span><span class="n">check_for_path</span><span class="p">(</span><span class="n">hdfs_path</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="p">)</span>
-
-
-<span class="k">class</span> <span class="nc">S3KeySensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Waits for a key (a file-like instance on S3) to be present in a S3 bucket.</span>
-<span class="sd"> S3 being a key/value it does not support folders. The path is just a key</span>
-<span class="sd"> a resource.</span>
-
-<span class="sd"> :param bucket_key: The key being waited on. Supports full s3:// style url</span>
-<span class="sd"> or relative path from root level.</span>
-<span class="sd"> :type bucket_key: str</span>
-<span class="sd"> :param bucket_name: Name of the S3 bucket</span>
-<span class="sd"> :type bucket_name: str</span>
-<span class="sd"> :param wildcard_match: whether the bucket_key should be interpreted as a</span>
-<span class="sd"> Unix wildcard pattern</span>
-<span class="sd"> :type wildcard_match: bool</span>
-<span class="sd"> :param aws_conn_id: a reference to the s3 connection</span>
-<span class="sd"> :type aws_conn_id: str</span>
-<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bucket_key'</span><span class="p">,</span> <span class="s1">'bucket_name'</span><span class="p">)</span>
-
- <span class="nd">@apply_defaults</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
- <span class="bp">self</span><span class="p">,</span> <span class="n">bucket_key</span><span class="p">,</span>
- <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">wildcard_match</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">aws_conn_id</span><span class="o">=</span><span class="s1">'aws_default'</span><span class="p">,</span>
- <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">S3KeySensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
- <span class="c1"># Parse</span>
- <span class="k">if</span> <span class="n">bucket_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
- <span class="n">parsed_url</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">bucket_key</span><span class="p">)</span>
- <span class="k">if</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span> <span class="o">==</span> <span class="s1">''</span><span class="p">:</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Please provide a bucket_name'</span><span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">bucket_name</span> <span class="o">=</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span>
- <span class="k">if</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">path</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'/'</span><span class="p">:</span>
- <span class="n">bucket_key</span> <span class="o">=</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">path</span><span class="p">[</span><span class="mi">1</span><span class="p">:]</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">bucket_key</span> <span class="o">=</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">path</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">bucket_name</span> <span class="o">=</span> <span class="n">bucket_name</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">bucket_key</span> <span class="o">=</span> <span class="n">bucket_key</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">wildcard_match</span> <span class="o">=</span> <span class="n">wildcard_match</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span> <span class="o">=</span> <span class="n">aws_conn_id</span>
-
- <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
- <span class="kn">from</span> <span class="nn">airflow.hooks.S3_hook</span> <span class="k">import</span> <span class="n">S3Hook</span>
- <span class="n">hook</span> <span class="o">=</span> <span class="n">S3Hook</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span>
- <span class="n">full_url</span> <span class="o">=</span> <span class="s2">"s3://"</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket_name</span> <span class="o">+</span> <span class="s2">"/"</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket_key</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Poking for key : </span><span class="si">{full_url}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">wildcard_match</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">check_for_wildcard_key</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket_key</span><span class="p">,</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">bucket_name</span><span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">check_for_key</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket_key</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket_name</span><span class="p">)</span>
-
-
-<span class="k">class</span> <span class="nc">S3PrefixSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Waits for a prefix to exist. A prefix is the first part of a key,</span>
-<span class="sd"> thus enabling checking of constructs similar to glob airfl* or</span>
-<span class="sd"> SQL LIKE 'airfl%'. There is the possibility to precise a delimiter to</span>
-<span class="sd"> indicate the hierarchy or keys, meaning that the match will stop at that</span>
-<span class="sd"> delimiter. Current code accepts sane delimiters, i.e. characters that</span>
-<span class="sd"> are NOT special characters in the Python regex engine.</span>
-
-<span class="sd"> :param bucket_name: Name of the S3 bucket</span>
-<span class="sd"> :type bucket_name: str</span>
-<span class="sd"> :param prefix: The prefix being waited on. Relative path from bucket root level.</span>
-<span class="sd"> :type prefix: str</span>
-<span class="sd"> :param delimiter: The delimiter intended to show hierarchy.</span>
-<span class="sd"> Defaults to '/'.</span>
-<span class="sd"> :type delimiter: str</span>
-<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'prefix'</span><span class="p">,</span> <span class="s1">'bucket_name'</span><span class="p">)</span>
-
- <span class="nd">@apply_defaults</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
- <span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span>
- <span class="n">prefix</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">'/'</span><span class="p">,</span>
- <span class="n">aws_conn_id</span><span class="o">=</span><span class="s1">'aws_default'</span><span class="p">,</span>
- <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">S3PrefixSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
- <span class="c1"># Parse</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">bucket_name</span> <span class="o">=</span> <span class="n">bucket_name</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">prefix</span> <span class="o">=</span> <span class="n">prefix</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">delimiter</span> <span class="o">=</span> <span class="n">delimiter</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">full_url</span> <span class="o">=</span> <span class="s2">"s3://"</span> <span class="o">+</span> <span class="n">bucket_name</span> <span class="o">+</span> <span class="s1">'/'</span> <span class="o">+</span> <span class="n">prefix</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span> <span class="o">=</span> <span class="n">aws_conn_id</span>
-
- <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Poking for prefix : </span><span class="si">{self.prefix}</span><span class="se">\n</span><span class="s1">'</span>
- <span class="s1">'in bucket s3://</span><span class="si">{self.bucket_name}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
- <span class="kn">from</span> <span class="nn">airflow.hooks.S3_hook</span> <span class="k">import</span> <span class="n">S3Hook</span>
- <span class="n">hook</span> <span class="o">=</span> <span class="n">S3Hook</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">check_for_prefix</span><span class="p">(</span>
- <span class="n">prefix</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">prefix</span><span class="p">,</span>
- <span class="n">delimiter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delimiter</span><span class="p">,</span>
- <span class="n">bucket_name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket_name</span><span class="p">)</span>
-
-
-<span class="k">class</span> <span class="nc">TimeSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Waits until the specified time of the day.</span>
-
-<span class="sd"> :param target_time: time after which the job succeeds</span>
-<span class="sd"> :type target_time: datetime.time</span>
-<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">()</span>
-
- <span class="nd">@apply_defaults</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">target_time</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">TimeSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">target_time</span> <span class="o">=</span> <span class="n">target_time</span>
-
- <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Checking if the time (</span><span class="si">%s</span><span class="s1">) has come'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">target_time</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span><span class="o">.</span><span class="n">time</span><span class="p">()</span> <span class="o">></span> <span class="bp">self</span><span class="o">.</span><span class="n">target_time</span>
-
-
-<span class="k">class</span> <span class="nc">TimeDeltaSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Waits for a timedelta after the task's execution_date + schedule_interval.</span>
-<span class="sd"> In Airflow, the daily task stamped with ``execution_date``</span>
-<span class="sd"> 2016-01-01 can only start running on 2016-01-02. The timedelta here</span>
-<span class="sd"> represents the time after the execution period has closed.</span>
-
-<span class="sd"> :param delta: time length to wait after execution_date before succeeding</span>
-<span class="sd"> :type delta: datetime.timedelta</span>
-<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">()</span>
-
- <span class="nd">@apply_defaults</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">delta</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">TimeDeltaSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">delta</span> <span class="o">=</span> <span class="n">delta</span>
-
- <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
- <span class="n">dag</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">'dag'</span><span class="p">]</span>
- <span class="n">target_dttm</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">following_schedule</span><span class="p">(</span><span class="n">context</span><span class="p">[</span><span class="s1">'execution_date'</span><span class="p">])</span>
- <span class="n">target_dttm</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">delta</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Checking if the time (</span><span class="si">%s</span><span class="s1">) has come'</span><span class="p">,</span> <span class="n">target_dttm</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">></span> <span class="n">target_dttm</span>
-
-
-<span class="k">class</span> <span class="nc">HttpSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Executes a HTTP get statement and returns False on failure:</span>
-<span class="sd"> 404 not found or response_check function returned False</span>
-
-<span class="sd"> :param http_conn_id: The connection to run the sensor against</span>
-<span class="sd"> :type http_conn_id: string</span>
-<span class="sd"> :param method: The HTTP request method to use</span>
-<span class="sd"> :type method: string</span>
-<span class="sd"> :param endpoint: The relative part of the full url</span>
-<span class="sd"> :type endpoint: string</span>
-<span class="sd"> :param request_params: The parameters to be added to the GET url</span>
-<span class="sd"> :type request_params: a dictionary of string key/value pairs</span>
-<span class="sd"> :param headers: The HTTP headers to be added to the GET request</span>
-<span class="sd"> :type headers: a dictionary of string key/value pairs</span>
-<span class="sd"> :param response_check: A check against the 'requests' response object.</span>
-<span class="sd"> Returns True for 'pass' and False otherwise.</span>
-<span class="sd"> :type response_check: A lambda or defined function.</span>
-<span class="sd"> :param extra_options: Extra options for the 'requests' library, see the</span>
-<span class="sd"> 'requests' documentation (options to modify timeout, ssl, etc.)</span>
-<span class="sd"> :type extra_options: A dictionary of options, where key is string and value</span>
-<span class="sd"> depends on the option that's being modified.</span>
-<span class="sd"> ""&quo
<TRUNCATED>
[51/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
1.10.0
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/commit/11437c14
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/tree/11437c14
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/diff/11437c14
Branch: refs/heads/asf-site
Commit: 11437c14a3607f6b85b6610dc776f55d78e7d767
Parents: 28a3eb6
Author: Kaxil Naik <ka...@gmail.com>
Authored: Mon Aug 27 17:22:22 2018 +0100
Committer: Kaxil Naik <ka...@gmail.com>
Committed: Mon Aug 27 17:22:22 2018 +0100
----------------------------------------------------------------------
_images/connection_create.png | Bin 0 -> 41547 bytes
_images/connection_edit.png | Bin 0 -> 53636 bytes
_images/connections.png | Bin 93057 -> 48442 bytes
_modules/S3_hook.html | 489 -
.../contrib/executors/mesos_executor.html | 89 +-
.../contrib/hooks/aws_dynamodb_hook.html | 300 +
_modules/airflow/contrib/hooks/aws_hook.html | 410 +
.../airflow/contrib/hooks/aws_lambda_hook.html | 302 +
.../airflow/contrib/hooks/bigquery_hook.html | 889 +-
.../airflow/contrib/hooks/databricks_hook.html | 462 +
.../airflow/contrib/hooks/datadog_hook.html | 375 +
.../airflow/contrib/hooks/datastore_hook.html | 52 +-
.../contrib/hooks/discord_webhook_hook.html | 375 +
_modules/airflow/contrib/hooks/emr_hook.html | 35 +-
_modules/airflow/contrib/hooks/fs_hook.html | 281 +
_modules/airflow/contrib/hooks/ftp_hook.html | 494 +
.../contrib/hooks/gcp_api_base_hook.html | 379 +
.../contrib/hooks/gcp_dataflow_hook.html | 196 +-
.../contrib/hooks/gcp_dataproc_hook.html | 463 +
.../contrib/hooks/gcp_mlengine_hook.html | 20 +-
.../airflow/contrib/hooks/gcp_pubsub_hook.html | 519 +
_modules/airflow/contrib/hooks/gcs_hook.html | 315 +-
.../airflow/contrib/hooks/jenkins_hook.html | 283 +
_modules/airflow/contrib/hooks/jira_hook.html | 319 +
_modules/airflow/contrib/hooks/pinot_hook.html | 340 +
_modules/airflow/contrib/hooks/qubole_hook.html | 449 +
_modules/airflow/contrib/hooks/redis_hook.html | 328 +
.../airflow/contrib/hooks/redshift_hook.html | 348 +
_modules/airflow/contrib/hooks/sftp_hook.html | 404 +
.../contrib/hooks/slack_webhook_hook.html | 364 +
.../airflow/contrib/hooks/spark_jdbc_hook.html | 481 +
.../airflow/contrib/hooks/spark_sql_hook.html | 396 +
.../contrib/hooks/spark_submit_hook.html | 799 ++
_modules/airflow/contrib/hooks/sqoop_hook.html | 580 +
_modules/airflow/contrib/hooks/ssh_hook.html | 470 +
.../airflow/contrib/hooks/vertica_hook.html | 288 +
_modules/airflow/contrib/hooks/wasb_hook.html | 94 +-
_modules/airflow/contrib/kubernetes/secret.html | 276 +
.../contrib/operators/awsbatch_operator.html | 403 +
.../operators/bigquery_check_operator.html | 37 +-
.../contrib/operators/bigquery_get_data.html | 351 +
.../contrib/operators/bigquery_operator.html | 399 +-
.../bigquery_table_delete_operator.html | 301 +
.../contrib/operators/bigquery_to_bigquery.html | 39 +-
.../contrib/operators/bigquery_to_gcs.html | 39 +-
.../contrib/operators/databricks_operator.html | 45 +-
.../contrib/operators/dataflow_operator.html | 241 +-
.../contrib/operators/dataproc_operator.html | 866 +-
.../operators/datastore_export_operator.html | 344 +
.../operators/datastore_import_operator.html | 332 +
.../operators/discord_webhook_operator.html | 333 +
.../airflow/contrib/operators/ecs_operator.html | 31 +-
.../operators/emr_add_steps_operator.html | 33 +-
.../operators/emr_create_job_flow_operator.html | 33 +-
.../emr_terminate_job_flow_operator.html | 31 +-
.../airflow/contrib/operators/file_to_gcs.html | 310 +
.../airflow/contrib/operators/file_to_wasb.html | 35 +-
.../operators/gcs_download_operator.html | 56 +-
.../contrib/operators/gcs_list_operator.html | 326 +
.../airflow/contrib/operators/gcs_operator.html | 357 +
.../airflow/contrib/operators/gcs_to_bq.html | 288 +-
.../airflow/contrib/operators/gcs_to_gcs.html | 365 +
.../airflow/contrib/operators/gcs_to_s3.html | 347 +
.../contrib/operators/hipchat_operator.html | 35 +-
.../operators/jenkins_job_trigger_operator.html | 484 +
.../contrib/operators/jira_operator.html | 329 +
.../operators/kubernetes_pod_operator.html | 362 +
.../contrib/operators/mlengine_operator.html | 260 +-
.../airflow/contrib/operators/mysql_to_gcs.html | 524 +
.../operators/postgres_to_gcs_operator.html | 481 +
.../contrib/operators/pubsub_operator.html | 669 ++
.../contrib/operators/qubole_operator.html | 411 +
.../contrib/operators/s3_list_operator.html | 317 +
.../contrib/operators/s3_to_gcs_operator.html | 425 +
.../contrib/operators/sftp_operator.html | 336 +
.../operators/slack_webhook_operator.html | 327 +
.../contrib/operators/spark_jdbc_operator.html | 449 +
.../contrib/operators/spark_sql_operator.html | 344 +
.../operators/spark_submit_operator.html | 405 +
.../contrib/operators/sqoop_operator.html | 467 +
.../airflow/contrib/operators/ssh_operator.html | 394 +
.../contrib/operators/vertica_operator.html | 284 +
.../sensors/aws_redshift_cluster_sensor.html | 287 +
.../airflow/contrib/sensors/bash_sensor.html | 331 +
.../contrib/sensors/bigquery_sensor.html | 306 +
.../airflow/contrib/sensors/datadog_sensor.html | 318 +
.../contrib/sensors/emr_base_sensor.html | 291 +
.../contrib/sensors/emr_job_flow_sensor.html | 288 +
.../contrib/sensors/emr_step_sensor.html | 292 +
.../airflow/contrib/sensors/file_sensor.html | 306 +
.../airflow/contrib/sensors/ftp_sensor.html | 302 +
.../airflow/contrib/sensors/gcs_sensor.html | 408 +
.../airflow/contrib/sensors/hdfs_sensor.html | 313 +
.../airflow/contrib/sensors/jira_sensor.html | 382 +
.../airflow/contrib/sensors/pubsub_sensor.html | 340 +
.../airflow/contrib/sensors/qubole_sensor.html | 323 +
.../contrib/sensors/redis_key_sensor.html | 282 +
.../airflow/contrib/sensors/sftp_sensor.html | 287 +
.../airflow/contrib/sensors/wasb_sensor.html | 33 +-
_modules/airflow/executors/celery_executor.html | 201 +-
_modules/airflow/executors/local_executor.html | 240 +-
.../airflow/executors/sequential_executor.html | 37 +-
_modules/airflow/hooks/S3_hook.html | 200 +-
_modules/airflow/hooks/dbapi_hook.html | 515 +
_modules/airflow/hooks/hdfs_hook.html | 335 +
_modules/airflow/hooks/http_hook.html | 358 +
_modules/airflow/hooks/mssql_hook.html | 287 +
_modules/airflow/hooks/mysql_hook.html | 341 +
_modules/airflow/hooks/oracle_hook.html | 382 +
_modules/airflow/hooks/pig_hook.html | 329 +
_modules/airflow/hooks/postgres_hook.html | 323 +
_modules/airflow/hooks/presto_hook.html | 366 +
_modules/airflow/hooks/slack_hook.html | 296 +
_modules/airflow/hooks/sqlite_hook.html | 276 +
_modules/airflow/hooks/webhdfs_hook.html | 344 +
_modules/airflow/macros.html | 31 +-
_modules/airflow/macros/hive.html | 52 +-
_modules/airflow/models.html | 1004 +-
_modules/airflow/operators/bash_operator.html | 376 +
_modules/airflow/operators/check_operator.html | 497 +
_modules/airflow/operators/dagrun_operator.html | 317 +
_modules/airflow/operators/docker_operator.html | 469 -
_modules/airflow/operators/dummy_operator.html | 273 +
_modules/airflow/operators/email_operator.html | 313 +
.../airflow/operators/generic_transfer.html | 315 +
_modules/airflow/operators/http_operator.html | 327 +
.../airflow/operators/latest_only_operator.html | 300 +
_modules/airflow/operators/mssql_operator.html | 291 +
_modules/airflow/operators/mysql_operator.html | 295 +
_modules/airflow/operators/oracle_operator.html | 290 +
_modules/airflow/operators/pig_operator.html | 307 +
.../airflow/operators/postgres_operator.html | 297 +
.../operators/presto_check_operator.html | 361 +
_modules/airflow/operators/presto_to_mysql.html | 314 +
_modules/airflow/operators/python_operator.html | 613 +
.../operators/redshift_to_s3_operator.html | 118 +-
.../operators/s3_file_transform_operator.html | 96 +-
.../airflow/operators/s3_to_hive_operator.html | 482 -
.../operators/s3_to_redshift_operator.html | 330 +
_modules/airflow/operators/sensors.html | 929 --
_modules/airflow/operators/slack_operator.html | 366 +
_modules/airflow/operators/sqlite_operator.html | 286 +
_modules/airflow/operators/subdag_operator.html | 338 +
.../airflow/sensors/base_sensor_operator.html | 310 +
.../airflow/sensors/external_task_sensor.html | 336 +
_modules/airflow/sensors/hdfs_sensor.html | 352 +
.../airflow/sensors/hive_partition_sensor.html | 310 +
_modules/airflow/sensors/http_sensor.html | 327 +
.../sensors/metastore_partition_sensor.html | 318 +
.../sensors/named_hive_partition_sensor.html | 339 +
_modules/airflow/sensors/s3_key_sensor.html | 316 +
_modules/airflow/sensors/s3_prefix_sensor.html | 303 +
_modules/airflow/sensors/sql_sensor.html | 293 +
_modules/airflow/sensors/time_delta_sensor.html | 281 +
_modules/airflow/sensors/time_sensor.html | 275 +
_modules/airflow/sensors/web_hdfs_sensor.html | 279 +
_modules/bash_operator.html | 350 -
_modules/bigquery_hook.html | 1279 --
_modules/cloudant_hook.html | 275 -
_modules/dagrun_operator.html | 310 -
_modules/dbapi_hook.html | 499 -
_modules/druid_hook.html | 325 -
_modules/dummy_operator.html | 266 -
_modules/email_operator.html | 297 -
_modules/ftp_hook.html | 486 -
_modules/gcs_hook.html | 498 -
_modules/generic_transfer.html | 308 -
_modules/hive_hooks.html | 996 --
_modules/hive_operator.html | 339 -
_modules/hive_to_druid.html | 454 -
_modules/hive_to_mysql.html | 339 -
_modules/hive_to_samba_operator.html | 260 -
_modules/http_hook.html | 351 -
_modules/http_operator.html | 320 -
_modules/index.html | 155 +-
_modules/mssql_hook.html | 242 -
_modules/mssql_operator.html | 247 -
_modules/mssql_to_hive.html | 326 -
_modules/mysql_hook.html | 334 -
_modules/mysql_operator.html | 288 -
_modules/mysql_to_hive.html | 368 -
_modules/postgres_hook.html | 316 -
_modules/postgres_operator.html | 288 -
_modules/presto_check_operator.html | 354 -
_modules/presto_hook.html | 346 -
_modules/python_operator.html | 591 -
_modules/qubole_operator.html | 404 -
_modules/s3_to_hive_operator.html | 482 -
_modules/sensors.html | 929 --
_modules/slack_operator.html | 352 -
_modules/sqlite_hook.html | 269 -
_modules/ssh_execute_operator.html | 387 -
_modules/ssh_hook.html | 386 -
_modules/vertica_hook.html | 247 -
_modules/vertica_operator.html | 246 -
_modules/vertica_to_hive.html | 330 -
_modules/webhdfs_hook.html | 300 -
_sources/api.rst.txt | 13 +-
_sources/cli.txt | 11 -
_sources/code.rst.txt | 336 +-
_sources/code.txt | 252 -
_sources/concepts.rst.txt | 49 +-
_sources/concepts.txt | 759 --
_sources/configuration.rst.txt | 310 -
_sources/configuration.txt | 230 -
_sources/faq.rst.txt | 43 +-
_sources/faq.txt | 142 -
_sources/howto/executor/use-celery.rst.txt | 47 +
_sources/howto/executor/use-dask.rst.txt | 33 +
_sources/howto/executor/use-mesos.rst.txt | 65 +
_sources/howto/index.rst.txt | 23 +
_sources/howto/initialize-database.rst.txt | 31 +
_sources/howto/manage-connections.rst.txt | 135 +
_sources/howto/run-with-systemd.rst.txt | 15 +
_sources/howto/run-with-upstart.rst.txt | 20 +
_sources/howto/secure-connections.rst.txt | 32 +
_sources/howto/set-config.rst.txt | 35 +
_sources/howto/use-test-config.rst.txt | 17 +
_sources/howto/write-logs.rst.txt | 159 +
_sources/index.rst.txt | 4 +-
_sources/index.txt | 87 -
_sources/installation.rst.txt | 29 +-
_sources/installation.txt | 90 -
_sources/integration.rst.txt | 428 +-
_sources/kubernetes.rst.txt | 101 +
_sources/license.rst.txt | 35 +-
_sources/license.txt | 211 -
_sources/lineage.rst.txt | 85 +
_sources/plugins.rst.txt | 18 +-
_sources/plugins.txt | 139 -
_sources/profiling.rst.txt | 2 +-
_sources/profiling.txt | 39 -
_sources/project.rst.txt | 7 +
_sources/project.txt | 59 -
_sources/scheduler.rst.txt | 4 +-
_sources/scheduler.txt | 101 -
_sources/security.rst.txt | 22 +-
_sources/security.txt | 253 -
_sources/start.rst.txt | 7 +-
_sources/start.txt | 49 -
_sources/timezone.rst.txt | 143 +
_sources/tutorial.rst.txt | 4 +-
_sources/tutorial.txt | 429 -
_sources/ui.txt | 102 -
_static/apache.jpg | Bin 43364 -> 0 bytes
_static/basic.css | 2 +-
_static/default.css | 3 -
_static/doctools.js | 4 +-
_static/fonts/Inconsolata.ttf | Bin 63184 -> 0 bytes
_static/incubator.jpg | Bin 91227 -> 0 bytes
_static/jquery-1.11.1.js | 10308 -----------------
_static/searchtools.js | 2 +-
_static/websupport.js | 2 +-
api.html | 18 +-
cli.html | 256 +-
code.html | 10181 ++++++++++++----
concepts.html | 50 +-
configuration.html | 520 -
faq.html | 50 +-
genindex.html | 938 +-
howto/executor/use-celery.html | 301 +
howto/executor/use-dask.html | 287 +
howto/executor/use-mesos.html | 326 +
howto/index.html | 295 +
howto/initialize-database.html | 292 +
howto/manage-connections.html | 381 +
howto/run-with-systemd.html | 274 +
howto/run-with-upstart.html | 276 +
howto/secure-connections.html | 290 +
howto/set-config.html | 289 +
howto/use-test-config.html | 275 +
howto/write-logs.html | 430 +
index.html | 163 +-
installation.html | 32 +-
integration.html | 2801 ++++-
kubernetes.html | 369 +
license.html | 381 +-
lineage.html | 328 +
objects.inv | Bin 3480 -> 6013 bytes
plugins.html | 22 +-
profiling.html | 6 +-
project.html | 11 +-
py-modindex.html | 34 +-
scheduler.html | 8 +-
search.html | 4 +-
searchindex.js | 2 +-
security.html | 41 +-
start.html | 11 +-
timezone.html | 374 +
tutorial.html | 12 +-
ui.html | 8 +-
291 files changed, 63681 insertions(+), 38174 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_images/connection_create.png
----------------------------------------------------------------------
diff --git a/_images/connection_create.png b/_images/connection_create.png
new file mode 100644
index 0000000..8a574d4
Binary files /dev/null and b/_images/connection_create.png differ
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_images/connection_edit.png
----------------------------------------------------------------------
diff --git a/_images/connection_edit.png b/_images/connection_edit.png
new file mode 100644
index 0000000..c6d14da
Binary files /dev/null and b/_images/connection_edit.png differ
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_images/connections.png
----------------------------------------------------------------------
diff --git a/_images/connections.png b/_images/connections.png
index d07a130..3a28473 100644
Binary files a/_images/connections.png and b/_images/connections.png differ
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/S3_hook.html
----------------------------------------------------------------------
diff --git a/_modules/S3_hook.html b/_modules/S3_hook.html
deleted file mode 100644
index ae1091a..0000000
--- a/_modules/S3_hook.html
+++ /dev/null
@@ -1,489 +0,0 @@
-
-
-<!DOCTYPE html>
-<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
-<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
-<head>
- <meta charset="utf-8">
-
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
-
- <title>S3_hook — Airflow Documentation</title>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../genindex.html"/>
- <link rel="search" title="Search" href="../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../index.html"/>
- <link rel="up" title="Module code" href="index.html"/>
-
-
- <script src="../_static/js/modernizr.min.js"></script>
-
-</head>
-
-<body class="wy-body-for-nav" role="document">
-
-
- <div class="wy-grid-for-nav">
-
-
- <nav data-toggle="wy-nav-shift" class="wy-nav-side">
- <div class="wy-side-scroll">
- <div class="wy-side-nav-search">
-
-
-
- <a href="../index.html" class="icon icon-home"> Airflow
-
-
-
- </a>
-
-
-
-
-
-
-
-<div role="search">
- <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
- <input type="text" name="q" placeholder="Search docs" />
- <input type="hidden" name="check_keywords" value="yes" />
- <input type="hidden" name="area" value="default" />
- </form>
-</div>
-
-
- </div>
-
- <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
-
-
-
-
-
-
- <ul>
-<li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../configuration.html">Configuration</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../ui.html">UI / Screenshots</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../concepts.html">Concepts</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../profiling.html">Data Profiling</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../cli.html">Command Line Interface</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../scheduler.html">Scheduling & Triggers</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../plugins.html">Plugins</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../api.html">Experimental Rest API</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../integration.html">Integration</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../code.html">API Reference</a></li>
-</ul>
-
-
-
- </div>
- </div>
- </nav>
-
- <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
-
-
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
-
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="../index.html">Airflow</a>
-
- </nav>
-
-
-
- <div class="wy-nav-content">
- <div class="rst-content">
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-<div role="navigation" aria-label="breadcrumbs navigation">
-
- <ul class="wy-breadcrumbs">
-
- <li><a href="../index.html">Docs</a> »</li>
-
- <li><a href="index.html">Module code</a> »</li>
-
- <li>S3_hook</li>
-
-
- <li class="wy-breadcrumbs-aside">
-
-
-
- </li>
-
- </ul>
-
-
- <hr/>
-</div>
- <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
- <div itemprop="articleBody">
-
- <h1>Source code for S3_hook</h1><div class="highlight"><pre>
-<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
-<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
-
-<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
-<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.aws_hook</span> <span class="k">import</span> <span class="n">AwsHook</span>
-
-<span class="kn">from</span> <span class="nn">six</span> <span class="k">import</span> <span class="n">BytesIO</span>
-<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="k">import</span> <span class="n">urlparse</span>
-<span class="kn">import</span> <span class="nn">re</span>
-<span class="kn">import</span> <span class="nn">fnmatch</span>
-
-<div class="viewcode-block" id="S3Hook"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook">[docs]</a><span class="k">class</span> <span class="nc">S3Hook</span><span class="p">(</span><span class="n">AwsHook</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Interact with AWS S3, using the boto3 library.</span>
-<span class="sd"> """</span>
-
- <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_client_type</span><span class="p">(</span><span class="s1">'s3'</span><span class="p">)</span>
-
- <span class="nd">@staticmethod</span>
- <span class="k">def</span> <span class="nf">parse_s3_url</span><span class="p">(</span><span class="n">s3url</span><span class="p">):</span>
- <span class="n">parsed_url</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">s3url</span><span class="p">)</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span><span class="p">:</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Please provide a bucket_name'</span><span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">bucket_name</span> <span class="o">=</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span>
- <span class="n">key</span> <span class="o">=</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">'/'</span><span class="p">)</span>
- <span class="k">return</span> <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span>
-
-<div class="viewcode-block" id="S3Hook.check_for_bucket"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.check_for_bucket">[docs]</a> <span class="k">def</span> <span class="nf">check_for_bucket</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Check if bucket_name exists.</span>
-
-<span class="sd"> :param bucket_name: the name of the bucket</span>
-<span class="sd"> :type bucket_name: str</span>
-<span class="sd"> """</span>
- <span class="k">try</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">head_bucket</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">)</span>
- <span class="k">return</span> <span class="kc">True</span>
- <span class="k">except</span><span class="p">:</span>
- <span class="k">return</span> <span class="kc">False</span></div>
-
-<div class="viewcode-block" id="S3Hook.get_bucket"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.get_bucket">[docs]</a> <span class="k">def</span> <span class="nf">get_bucket</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Returns a boto3.S3.Bucket object</span>
-
-<span class="sd"> :param bucket_name: the name of the bucket</span>
-<span class="sd"> :type bucket_name: str</span>
-<span class="sd"> """</span>
- <span class="n">s3</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_resource_type</span><span class="p">(</span><span class="s1">'s3'</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">s3</span><span class="o">.</span><span class="n">Bucket</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">)</span></div>
-
-<div class="viewcode-block" id="S3Hook.check_for_prefix"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.check_for_prefix">[docs]</a> <span class="k">def</span> <span class="nf">check_for_prefix</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="p">,</span> <span class="n">delimiter</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Checks that a prefix exists in a bucket</span>
-<span class="sd"> """</span>
- <span class="n">prefix</span> <span class="o">=</span> <span class="n">prefix</span> <span class="o">+</span> <span class="n">delimiter</span> <span class="k">if</span> <span class="n">prefix</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="o">!=</span> <span class="n">delimiter</span> <span class="k">else</span> <span class="n">prefix</span>
- <span class="n">prefix_split</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="sa">r</span><span class="s1">'(\w+[</span><span class="si">{d}</span><span class="s1">])$'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">d</span><span class="o">=</span><span class="n">delimiter</span><span class="p">),</span> <span class="n">prefix</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
- <span class="n">previous_level</span> <span class="o">=</span> <span class="n">prefix_split</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
- <span class="n">plist</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">list_prefixes</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">previous_level</span><span class="p">,</span> <span class="n">delimiter</span><span class="p">)</span>
- <span class="k">return</span> <span class="kc">False</span> <span class="k">if</span> <span class="n">plist</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">prefix</span> <span class="ow">in</span> <span class="n">plist</span></div>
-
-<div class="viewcode-block" id="S3Hook.list_prefixes"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.list_prefixes">[docs]</a> <span class="k">def</span> <span class="nf">list_prefixes</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">''</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Lists prefixes in a bucket under prefix</span>
-
-<span class="sd"> :param bucket_name: the name of the bucket</span>
-<span class="sd"> :type bucket_name: str</span>
-<span class="sd"> :param prefix: a key prefix</span>
-<span class="sd"> :type prefix: str</span>
-<span class="sd"> :param delimiter: the delimiter marks key hierarchy.</span>
-<span class="sd"> :type delimiter: str</span>
-<span class="sd"> """</span>
- <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">list_objects_v2</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span>
- <span class="n">Prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">,</span>
- <span class="n">Delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span>
- <span class="k">return</span> <span class="p">[</span><span class="n">p</span><span class="p">[</span><span class="s1">'Prefix'</span><span class="p">]</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">response</span><span class="p">[</span><span class="s1">'CommonPrefixes'</span><span class="p">]]</span> <span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'CommonPrefixes'</span><span class="p">)</span> <span class="k">else</span> <span class="kc">None</span></div>
-
-<div class="viewcode-block" id="S3Hook.list_keys"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.list_keys">[docs]</a> <span class="k">def</span> <span class="nf">list_keys</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">''</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Lists keys in a bucket under prefix and not containing delimiter</span>
-
-<span class="sd"> :param bucket_name: the name of the bucket</span>
-<span class="sd"> :type bucket_name: str</span>
-<span class="sd"> :param prefix: a key prefix</span>
-<span class="sd"> :type prefix: str</span>
-<span class="sd"> :param delimiter: the delimiter marks key hierarchy.</span>
-<span class="sd"> :type delimiter: str</span>
-<span class="sd"> """</span>
- <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">list_objects_v2</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span>
- <span class="n">Prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">,</span>
- <span class="n">Delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span>
- <span class="k">return</span> <span class="p">[</span><span class="n">k</span><span class="p">[</span><span class="s1">'Key'</span><span class="p">]</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">response</span><span class="p">[</span><span class="s1">'Contents'</span><span class="p">]]</span> <span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Contents'</span><span class="p">)</span> <span class="k">else</span> <span class="kc">None</span></div>
-
-<div class="viewcode-block" id="S3Hook.check_for_key"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.check_for_key">[docs]</a> <span class="k">def</span> <span class="nf">check_for_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Checks if a key exists in a bucket</span>
-
-<span class="sd"> :param key: S3 key that will point to the file</span>
-<span class="sd"> :type key: str</span>
-<span class="sd"> :param bucket_name: Name of the bucket in which the file is stored</span>
-<span class="sd"> :type bucket_name: str</span>
-<span class="sd"> """</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">bucket_name</span><span class="p">:</span>
- <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
-
- <span class="k">try</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">head_object</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">Key</span><span class="o">=</span><span class="n">key</span><span class="p">)</span>
- <span class="k">return</span> <span class="kc">True</span>
- <span class="k">except</span><span class="p">:</span>
- <span class="k">return</span> <span class="kc">False</span></div>
-
-<div class="viewcode-block" id="S3Hook.get_key"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.get_key">[docs]</a> <span class="k">def</span> <span class="nf">get_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Returns a boto3.s3.Object</span>
-
-<span class="sd"> :param key: the path to the key</span>
-<span class="sd"> :type key: str</span>
-<span class="sd"> :param bucket_name: the name of the bucket</span>
-<span class="sd"> :type bucket_name: str</span>
-<span class="sd"> """</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">bucket_name</span><span class="p">:</span>
- <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
-
- <span class="n">obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_resource_type</span><span class="p">(</span><span class="s1">'s3'</span><span class="p">)</span><span class="o">.</span><span class="n">Object</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span>
- <span class="n">obj</span><span class="o">.</span><span class="n">load</span><span class="p">()</span>
- <span class="k">return</span> <span class="n">obj</span></div>
-
-<div class="viewcode-block" id="S3Hook.read_key"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.read_key">[docs]</a> <span class="k">def</span> <span class="nf">read_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Reads a key from S3</span>
-
-<span class="sd"> :param key: S3 key that will point to the file</span>
-<span class="sd"> :type key: str</span>
-<span class="sd"> :param bucket_name: Name of the bucket in which the file is stored</span>
-<span class="sd"> :type bucket_name: str</span>
-<span class="sd"> """</span>
-
- <span class="n">obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_key</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">obj</span><span class="o">.</span><span class="n">get</span><span class="p">()[</span><span class="s1">'Body'</span><span class="p">]</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span></div>
-
-<div class="viewcode-block" id="S3Hook.check_for_wildcard_key"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.check_for_wildcard_key">[docs]</a> <span class="k">def</span> <span class="nf">check_for_wildcard_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
- <span class="n">wildcard_key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">''</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Checks that a key matching a wildcard expression exists in a bucket</span>
-<span class="sd"> """</span>
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_wildcard_key</span><span class="p">(</span><span class="n">wildcard_key</span><span class="o">=</span><span class="n">wildcard_key</span><span class="p">,</span>
- <span class="n">bucket_name</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span>
- <span class="n">delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span></div>
-
-<div class="viewcode-block" id="S3Hook.get_wildcard_key"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.get_wildcard_key">[docs]</a> <span class="k">def</span> <span class="nf">get_wildcard_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">wildcard_key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">''</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Returns a boto3.s3.Object object matching the regular expression</span>
-
-<span class="sd"> :param regex_key: the path to the key</span>
-<span class="sd"> :type regex_key: str</span>
-<span class="sd"> :param bucket_name: the name of the bucket</span>
-<span class="sd"> :type bucket_name: str</span>
-<span class="sd"> """</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">bucket_name</span><span class="p">:</span>
- <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">wildcard_key</span><span class="p">)</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="n">wildcard_key</span><span class="p">)</span>
-
- <span class="n">prefix</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="sa">r</span><span class="s1">'[*]'</span><span class="p">,</span> <span class="n">wildcard_key</span><span class="p">,</span> <span class="mi">1</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
- <span class="n">klist</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">list_keys</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span>
- <span class="k">if</span> <span class="n">klist</span><span class="p">:</span>
- <span class="n">key_matches</span> <span class="o">=</span> <span class="p">[</span><span class="n">k</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">klist</span> <span class="k">if</span> <span class="n">fnmatch</span><span class="o">.</span><span class="n">fnmatch</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="n">wildcard_key</span><span class="p">)]</span>
- <span class="k">if</span> <span class="n">key_matches</span><span class="p">:</span>
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_key</span><span class="p">(</span><span class="n">key_matches</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">bucket_name</span><span class="p">)</span></div>
-
-<div class="viewcode-block" id="S3Hook.load_file"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.load_file">[docs]</a> <span class="k">def</span> <span class="nf">load_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
- <span class="n">filename</span><span class="p">,</span>
- <span class="n">key</span><span class="p">,</span>
- <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">replace</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">encrypt</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Loads a local file to S3</span>
-
-<span class="sd"> :param filename: name of the file to load.</span>
-<span class="sd"> :type filename: str</span>
-<span class="sd"> :param key: S3 key that will point to the file</span>
-<span class="sd"> :type key: str</span>
-<span class="sd"> :param bucket_name: Name of the bucket in which to store the file</span>
-<span class="sd"> :type bucket_name: str</span>
-<span class="sd"> :param replace: A flag to decide whether or not to overwrite the key</span>
-<span class="sd"> if it already exists. If replace is False and the key exists, an</span>
-<span class="sd"> error will be raised.</span>
-<span class="sd"> :type replace: bool</span>
-<span class="sd"> :param encrypt: If True, the file will be encrypted on the server-side</span>
-<span class="sd"> by S3 and will be stored in an encrypted form while at rest in S3.</span>
-<span class="sd"> :type encrypt: bool</span>
-<span class="sd"> """</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">bucket_name</span><span class="p">:</span>
- <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
-
- <span class="k">if</span> <span class="ow">not</span> <span class="n">replace</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">check_for_key</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">):</span>
- <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"The key </span><span class="si">{key}</span><span class="s2"> already exists."</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">))</span>
-
- <span class="n">extra_args</span><span class="o">=</span><span class="p">{}</span>
- <span class="k">if</span> <span class="n">encrypt</span><span class="p">:</span>
- <span class="n">extra_args</span><span class="p">[</span><span class="s1">'ServerSideEncryption'</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"AES256"</span>
-
- <span class="n">client</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
- <span class="n">client</span><span class="o">.</span><span class="n">upload_file</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">ExtraArgs</span><span class="o">=</span><span class="n">extra_args</span><span class="p">)</span></div>
-
-<div class="viewcode-block" id="S3Hook.load_string"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.load_string">[docs]</a> <span class="k">def</span> <span class="nf">load_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
- <span class="n">string_data</span><span class="p">,</span>
- <span class="n">key</span><span class="p">,</span>
- <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">replace</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">encrypt</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">encoding</span><span class="o">=</span><span class="s1">'utf-8'</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Loads a string to S3</span>
-
-<span class="sd"> This is provided as a convenience to drop a string in S3. It uses the</span>
-<span class="sd"> boto infrastructure to ship a file to s3. </span>
-
-<span class="sd"> :param string_data: string to set as content for the key.</span>
-<span class="sd"> :type string_data: str</span>
-<span class="sd"> :param key: S3 key that will point to the file</span>
-<span class="sd"> :type key: str</span>
-<span class="sd"> :param bucket_name: Name of the bucket in which to store the file</span>
-<span class="sd"> :type bucket_name: str</span>
-<span class="sd"> :param replace: A flag to decide whether or not to overwrite the key</span>
-<span class="sd"> if it already exists</span>
-<span class="sd"> :type replace: bool</span>
-<span class="sd"> :param encrypt: If True, the file will be encrypted on the server-side</span>
-<span class="sd"> by S3 and will be stored in an encrypted form while at rest in S3.</span>
-<span class="sd"> :type encrypt: bool</span>
-<span class="sd"> """</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">bucket_name</span><span class="p">:</span>
- <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
-
- <span class="k">if</span> <span class="ow">not</span> <span class="n">replace</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">check_for_key</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">):</span>
- <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"The key </span><span class="si">{key}</span><span class="s2"> already exists."</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">))</span>
-
- <span class="n">extra_args</span><span class="o">=</span><span class="p">{}</span>
- <span class="k">if</span> <span class="n">encrypt</span><span class="p">:</span>
- <span class="n">extra_args</span><span class="p">[</span><span class="s1">'ServerSideEncryption'</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"AES256"</span>
-
- <span class="n">filelike_buffer</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">(</span><span class="n">string_data</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">encoding</span><span class="p">))</span>
-
- <span class="n">client</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
- <span class="n">client</span><span class="o">.</span><span class="n">upload_fileobj</span><span class="p">(</span><span class="n">filelike_buffer</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">ExtraArgs</span><span class="o">=</span><span class="n">extra_args</span><span class="p">)</span></div></div>
-</pre></div>
-
- </div>
- <div class="articleComments">
-
- </div>
- </div>
- <footer>
-
-
- <hr/>
-
- <div role="contentinfo">
- <p>
-
- </p>
- </div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
-
-</footer>
-
- </div>
- </div>
-
- </section>
-
- </div>
-
-
-
-
-
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../',
- VERSION:'',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../_static/jquery.js"></script>
- <script type="text/javascript" src="../_static/underscore.js"></script>
- <script type="text/javascript" src="../_static/doctools.js"></script>
-
-
-
-
-
- <script type="text/javascript" src="../_static/js/theme.js"></script>
-
-
-
-
- <script type="text/javascript">
- jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
- });
- </script>
-
-
-</body>
-</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/executors/mesos_executor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/executors/mesos_executor.html b/_modules/airflow/contrib/executors/mesos_executor.html
index f23df55..312fffe 100644
--- a/_modules/airflow/contrib/executors/mesos_executor.html
+++ b/_modules/airflow/contrib/executors/mesos_executor.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,22 @@
<h1>Source code for airflow.contrib.executors.mesos_executor</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="kn">from</span> <span class="nn">future</span> <span class="k">import</span> <span class="n">standard_library</span>
@@ -206,9 +213,9 @@
<span class="k">def</span> <span class="nf">get_framework_name</span><span class="p">():</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FRAMEWORK_NAME'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FRAMEWORK_NAME'</span><span class="p">):</span>
<span class="k">return</span> <span class="n">DEFAULT_FRAMEWORK_NAME</span>
- <span class="k">return</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FRAMEWORK_NAME'</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FRAMEWORK_NAME'</span><span class="p">)</span>
<span class="c1"># AirflowMesosScheduler, implements Mesos Scheduler interface</span>
@@ -232,11 +239,16 @@
<span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span> <span class="o">=</span> <span class="n">task_mem</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_counter</span> <span class="o">=</span> <span class="mi">0</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DOCKER_IMAGE_SLAVE'</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mesos_slave_docker_image</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DOCKER_IMAGE_SLAVE'</span>
+ <span class="p">)</span>
<span class="k">def</span> <span class="nf">registered</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">frameworkId</span><span class="p">,</span> <span class="n">masterInfo</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler registered to Mesos with framework ID </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
- <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">)</span> <span class="ow">and</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">)</span> <span class="ow">and</span> \
+ <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span>
<span class="c1"># Import here to work around a circular import error</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">Connection</span>
@@ -322,6 +334,21 @@
<span class="n">command</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">cmd</span>
<span class="n">task</span><span class="o">.</span><span class="n">command</span><span class="o">.</span><span class="n">MergeFrom</span><span class="p">(</span><span class="n">command</span><span class="p">)</span>
+ <span class="c1"># If docker image for airflow is specified in config then pull that</span>
+ <span class="c1"># image before running the above airflow command</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mesos_slave_docker_image</span><span class="p">:</span>
+ <span class="n">network</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="o">.</span><span class="n">DockerInfo</span><span class="o">.</span><span class="n">Network</span><span class="o">.</span><span class="n">Value</span><span class="p">(</span><span class="s1">'BRIDGE'</span><span class="p">)</span>
+ <span class="n">docker</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="o">.</span><span class="n">DockerInfo</span><span class="p">(</span>
+ <span class="n">image</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mesos_slave_docker_image</span><span class="p">,</span>
+ <span class="n">force_pull_image</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">network</span><span class="o">=</span><span class="n">network</span>
+ <span class="p">)</span>
+ <span class="n">container</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="p">(</span>
+ <span class="nb">type</span><span class="o">=</span><span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="o">.</span><span class="n">DOCKER</span><span class="p">,</span>
+ <span class="n">docker</span><span class="o">=</span><span class="n">docker</span>
+ <span class="p">)</span>
+ <span class="n">task</span><span class="o">.</span><span class="n">container</span><span class="o">.</span><span class="n">MergeFrom</span><span class="p">(</span><span class="n">container</span><span class="p">)</span>
+
<span class="n">tasks</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">task</span><span class="p">)</span>
<span class="n">remainingCpus</span> <span class="o">-=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span>
@@ -371,28 +398,28 @@
<span class="n">framework</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">FrameworkInfo</span><span class="p">()</span>
<span class="n">framework</span><span class="o">.</span><span class="n">user</span> <span class="o">=</span> <span class="s1">''</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'MASTER'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'MASTER'</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Expecting mesos master URL for mesos executor"</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"mesos.master not provided for mesos executor"</span><span class="p">)</span>
- <span class="n">master</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'MASTER'</span><span class="p">)</span>
+ <span class="n">master</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'MASTER'</span><span class="p">)</span>
<span class="n">framework</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">get_framework_name</span><span class="p">()</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_CPU'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_CPU'</span><span class="p">):</span>
<span class="n">task_cpu</span> <span class="o">=</span> <span class="mi">1</span>
<span class="k">else</span><span class="p">:</span>
- <span class="n">task_cpu</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_CPU'</span><span class="p">)</span>
+ <span class="n">task_cpu</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_CPU'</span><span class="p">)</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_MEMORY'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_MEMORY'</span><span class="p">):</span>
<span class="n">task_memory</span> <span class="o">=</span> <span class="mi">256</span>
<span class="k">else</span><span class="p">:</span>
- <span class="n">task_memory</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_MEMORY'</span><span class="p">)</span>
+ <span class="n">task_memory</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_MEMORY'</span><span class="p">)</span>
- <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">):</span>
<span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="kc">True</span>
- <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span>
<span class="c1"># Import here to work around a circular import error</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">Connection</span>
@@ -404,7 +431,9 @@
<span class="c1"># Set the Framework ID to let the scheduler reconnect with running tasks.</span>
<span class="n">framework</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">connection</span><span class="o">.</span><span class="n">extra</span>
- <span class="n">framework</span><span class="o">.</span><span class="n">failover_timeout</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">)</span>
+ <span class="n">framework</span><span class="o">.</span><span class="n">failover_timeout</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span>
+ <span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span>
+ <span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="kc">False</span>
@@ -415,17 +444,17 @@
<span class="n">implicit_acknowledgements</span> <span class="o">=</span> <span class="mi">1</span>
- <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'AUTHENTICATE'</span><span class="p">):</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_PRINCIPAL'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'AUTHENTICATE'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_PRINCIPAL'</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Expecting authentication principal in the environment"</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"mesos.default_principal not provided in authenticated mode"</span><span class="p">)</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_SECRET'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_SECRET'</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Expecting authentication secret in the environment"</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"mesos.default_secret not provided in authenticated mode"</span><span class="p">)</span>
<span class="n">credential</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">Credential</span><span class="p">()</span>
- <span class="n">credential</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_PRINCIPAL'</span><span class="p">)</span>
- <span class="n">credential</span><span class="o">.</span><span class="n">secret</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_SECRET'</span><span class="p">)</span>
+ <span class="n">credential</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_PRINCIPAL'</span><span class="p">)</span>
+ <span class="n">credential</span><span class="o">.</span><span class="n">secret</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_SECRET'</span><span class="p">)</span>
<span class="n">framework</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="n">credential</span><span class="o">.</span><span class="n">principal</span>
@@ -446,7 +475,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span> <span class="o">=</span> <span class="n">driver</span>
<span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
- <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">executor_config</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
[32/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/datastore_export_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/datastore_export_operator.html b/_modules/airflow/contrib/operators/datastore_export_operator.html
new file mode 100644
index 0000000..94e2d02
--- /dev/null
+++ b/_modules/airflow/contrib/operators/datastore_export_operator.html
@@ -0,0 +1,344 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.datastore_export_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.datastore_export_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.datastore_export_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.datastore_hook</span> <span class="k">import</span> <span class="n">DatastoreHook</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcs_hook</span> <span class="k">import</span> <span class="n">GoogleCloudStorageHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="DatastoreExportOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.datastore_export_operator.DatastoreExportOperator">[docs]</a><span class="k">class</span> <span class="nc">DatastoreExportOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Export entities from Google Cloud Datastore to Cloud Storage</span>
+
+<span class="sd"> :param bucket: name of the cloud storage bucket to backup data</span>
+<span class="sd"> :type bucket: string</span>
+<span class="sd"> :param namespace: optional namespace path in the specified Cloud Storage bucket</span>
+<span class="sd"> to backup data. If this namespace does not exist in GCS, it will be created.</span>
+<span class="sd"> :type namespace: str</span>
+<span class="sd"> :param datastore_conn_id: the name of the Datastore connection id to use</span>
+<span class="sd"> :type datastore_conn_id: string</span>
+<span class="sd"> :param cloud_storage_conn_id: the name of the cloud storage connection id to force-write</span>
+<span class="sd"> backup</span>
+<span class="sd"> :type cloud_storage_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request must have domain-wide</span>
+<span class="sd"> delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> :param entity_filter: description of what data from the project is included in the export,</span>
+<span class="sd"> refer to https://cloud.google.com/datastore/docs/reference/rest/Shared.Types/EntityFilter</span>
+<span class="sd"> :type entity_filter: dict</span>
+<span class="sd"> :param labels: client-assigned labels for cloud storage</span>
+<span class="sd"> :type labels: dict</span>
+<span class="sd"> :param polling_interval_in_seconds: number of seconds to wait before polling for</span>
+<span class="sd"> execution status again</span>
+<span class="sd"> :type polling_interval_in_seconds: int</span>
+<span class="sd"> :param overwrite_existing: if the storage bucket + namespace is not empty, it will be</span>
+<span class="sd"> emptied prior to exports. This enables overwriting existing backups.</span>
+<span class="sd"> :type overwrite_existing: bool</span>
+<span class="sd"> :param xcom_push: push operation name to xcom for reference</span>
+<span class="sd"> :type xcom_push: bool</span>
+<span class="sd"> """</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">bucket</span><span class="p">,</span>
+ <span class="n">namespace</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">datastore_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">cloud_storage_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">entity_filter</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">polling_interval_in_seconds</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span>
+ <span class="n">overwrite_existing</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">xcom_push</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">DatastoreExportOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">datastore_conn_id</span> <span class="o">=</span> <span class="n">datastore_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cloud_storage_conn_id</span> <span class="o">=</span> <span class="n">cloud_storage_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">namespace</span> <span class="o">=</span> <span class="n">namespace</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">entity_filter</span> <span class="o">=</span> <span class="n">entity_filter</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="o">=</span> <span class="n">labels</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">polling_interval_in_seconds</span> <span class="o">=</span> <span class="n">polling_interval_in_seconds</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">overwrite_existing</span> <span class="o">=</span> <span class="n">overwrite_existing</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">xcom_push</span> <span class="o">=</span> <span class="n">xcom_push</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Exporting data to Cloud Storage bucket '</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">overwrite_existing</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">namespace</span><span class="p">:</span>
+ <span class="n">gcs_hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cloud_storage_conn_id</span><span class="p">)</span>
+ <span class="n">objects</span> <span class="o">=</span> <span class="n">gcs_hook</span><span class="o">.</span><span class="n">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">namespace</span><span class="p">)</span>
+ <span class="k">for</span> <span class="n">o</span> <span class="ow">in</span> <span class="n">objects</span><span class="p">:</span>
+ <span class="n">gcs_hook</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="n">o</span><span class="p">)</span>
+
+ <span class="n">ds_hook</span> <span class="o">=</span> <span class="n">DatastoreHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">datastore_conn_id</span><span class="p">,</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="n">ds_hook</span><span class="o">.</span><span class="n">export_to_storage_bucket</span><span class="p">(</span><span class="n">bucket</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span>
+ <span class="n">namespace</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">namespace</span><span class="p">,</span>
+ <span class="n">entity_filter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">entity_filter</span><span class="p">,</span>
+ <span class="n">labels</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">labels</span><span class="p">)</span>
+ <span class="n">operation_name</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="n">ds_hook</span><span class="o">.</span><span class="n">poll_operation_until_done</span><span class="p">(</span><span class="n">operation_name</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">polling_interval_in_seconds</span><span class="p">)</span>
+
+ <span class="n">state</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s1">'metadata'</span><span class="p">][</span><span class="s1">'common'</span><span class="p">][</span><span class="s1">'state'</span><span class="p">]</span>
+ <span class="k">if</span> <span class="n">state</span> <span class="o">!=</span> <span class="s1">'SUCCESSFUL'</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Operation failed: result=</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">result</span><span class="p">))</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">xcom_push</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">result</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/datastore_import_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/datastore_import_operator.html b/_modules/airflow/contrib/operators/datastore_import_operator.html
new file mode 100644
index 0000000..3b6d62c
--- /dev/null
+++ b/_modules/airflow/contrib/operators/datastore_import_operator.html
@@ -0,0 +1,332 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.datastore_import_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.datastore_import_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.datastore_import_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.datastore_hook</span> <span class="k">import</span> <span class="n">DatastoreHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="DatastoreImportOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.datastore_import_operator.DatastoreImportOperator">[docs]</a><span class="k">class</span> <span class="nc">DatastoreImportOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Import entities from Cloud Storage to Google Cloud Datastore</span>
+
+<span class="sd"> :param bucket: container in Cloud Storage to store data</span>
+<span class="sd"> :type bucket: string</span>
+<span class="sd"> :param file: path of the backup metadata file in the specified Cloud Storage bucket.</span>
+<span class="sd"> It should have the extension .overall_export_metadata</span>
+<span class="sd"> :type file: string</span>
+<span class="sd"> :param namespace: optional namespace of the backup metadata file in</span>
+<span class="sd"> the specified Cloud Storage bucket.</span>
+<span class="sd"> :type namespace: str</span>
+<span class="sd"> :param entity_filter: description of what data from the project is included in the export,</span>
+<span class="sd"> refer to https://cloud.google.com/datastore/docs/reference/rest/Shared.Types/EntityFilter</span>
+<span class="sd"> :type entity_filter: dict</span>
+<span class="sd"> :param labels: client-assigned labels for cloud storage</span>
+<span class="sd"> :type labels: dict</span>
+<span class="sd"> :param datastore_conn_id: the name of the connection id to use</span>
+<span class="sd"> :type datastore_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request must have domain-wide</span>
+<span class="sd"> delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> :param polling_interval_in_seconds: number of seconds to wait before polling for</span>
+<span class="sd"> execution status again</span>
+<span class="sd"> :type polling_interval_in_seconds: int</span>
+<span class="sd"> :param xcom_push: push operation name to xcom for reference</span>
+<span class="sd"> :type xcom_push: bool</span>
+<span class="sd"> """</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">bucket</span><span class="p">,</span>
+ <span class="n">file</span><span class="p">,</span>
+ <span class="n">namespace</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">entity_filter</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">datastore_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">polling_interval_in_seconds</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span>
+ <span class="n">xcom_push</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">DatastoreImportOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">datastore_conn_id</span> <span class="o">=</span> <span class="n">datastore_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">file</span> <span class="o">=</span> <span class="n">file</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">namespace</span> <span class="o">=</span> <span class="n">namespace</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">entity_filter</span> <span class="o">=</span> <span class="n">entity_filter</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="o">=</span> <span class="n">labels</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">polling_interval_in_seconds</span> <span class="o">=</span> <span class="n">polling_interval_in_seconds</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">xcom_push</span> <span class="o">=</span> <span class="n">xcom_push</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Importing data from Cloud Storage bucket </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">)</span>
+ <span class="n">ds_hook</span> <span class="o">=</span> <span class="n">DatastoreHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">datastore_conn_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="n">ds_hook</span><span class="o">.</span><span class="n">import_from_storage_bucket</span><span class="p">(</span><span class="n">bucket</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span>
+ <span class="n">file</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">file</span><span class="p">,</span>
+ <span class="n">namespace</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">namespace</span><span class="p">,</span>
+ <span class="n">entity_filter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">entity_filter</span><span class="p">,</span>
+ <span class="n">labels</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">labels</span><span class="p">)</span>
+ <span class="n">operation_name</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="n">ds_hook</span><span class="o">.</span><span class="n">poll_operation_until_done</span><span class="p">(</span><span class="n">operation_name</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">polling_interval_in_seconds</span><span class="p">)</span>
+
+ <span class="n">state</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s1">'metadata'</span><span class="p">][</span><span class="s1">'common'</span><span class="p">][</span><span class="s1">'state'</span><span class="p">]</span>
+ <span class="k">if</span> <span class="n">state</span> <span class="o">!=</span> <span class="s1">'SUCCESSFUL'</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Operation failed: result=</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">result</span><span class="p">))</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">xcom_push</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">result</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/discord_webhook_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/discord_webhook_operator.html b/_modules/airflow/contrib/operators/discord_webhook_operator.html
new file mode 100644
index 0000000..8aa057e
--- /dev/null
+++ b/_modules/airflow/contrib/operators/discord_webhook_operator.html
@@ -0,0 +1,333 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.discord_webhook_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.discord_webhook_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.discord_webhook_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.discord_webhook_hook</span> <span class="k">import</span> <span class="n">DiscordWebhookHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.operators.http_operator</span> <span class="k">import</span> <span class="n">SimpleHttpOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="DiscordWebhookOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.discord_webhook_operator.DiscordWebhookOperator">[docs]</a><span class="k">class</span> <span class="nc">DiscordWebhookOperator</span><span class="p">(</span><span class="n">SimpleHttpOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> This operator allows you to post messages to Discord using incoming webhooks.</span>
+<span class="sd"> Takes a Discord connection ID with a default relative webhook endpoint. The</span>
+<span class="sd"> default endpoint can be overridden using the webhook_endpoint parameter</span>
+<span class="sd"> (https://discordapp.com/developers/docs/resources/webhook).</span>
+
+<span class="sd"> Each Discord webhook can be pre-configured to use a specific username and</span>
+<span class="sd"> avatar_url. You can override these defaults in this operator.</span>
+
+<span class="sd"> :param http_conn_id: Http connection ID with host as "https://discord.com/api/" and</span>
+<span class="sd"> default webhook endpoint in the extra field in the form of</span>
+<span class="sd"> {"webhook_endpoint": "webhooks/{webhook.id}/{webhook.token}"}</span>
+<span class="sd"> :type http_conn_id: str</span>
+<span class="sd"> :param webhook_endpoint: Discord webhook endpoint in the form of</span>
+<span class="sd"> "webhooks/{webhook.id}/{webhook.token}"</span>
+<span class="sd"> :type webhook_endpoint: str</span>
+<span class="sd"> :param message: The message you want to send to your Discord channel</span>
+<span class="sd"> (max 2000 characters)</span>
+<span class="sd"> :type message: str</span>
+<span class="sd"> :param username: Override the default username of the webhook</span>
+<span class="sd"> :type username: str</span>
+<span class="sd"> :param avatar_url: Override the default avatar of the webhook</span>
+<span class="sd"> :type avatar_url: str</span>
+<span class="sd"> :param tts: Is a text-to-speech message</span>
+<span class="sd"> :type tts: bool</span>
+<span class="sd"> :param proxy: Proxy to use to make the Discord webhook call</span>
+<span class="sd"> :type proxy: str</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'username'</span><span class="p">,</span> <span class="s1">'message'</span><span class="p">)</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">http_conn_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">webhook_endpoint</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">message</span><span class="o">=</span><span class="s2">""</span><span class="p">,</span>
+ <span class="n">username</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">avatar_url</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">tts</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">proxy</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">DiscordWebhookOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">endpoint</span><span class="o">=</span><span class="n">webhook_endpoint</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">http_conn_id</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'No valid Discord http_conn_id supplied.'</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">http_conn_id</span> <span class="o">=</span> <span class="n">http_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">webhook_endpoint</span> <span class="o">=</span> <span class="n">webhook_endpoint</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">message</span> <span class="o">=</span> <span class="n">message</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="o">=</span> <span class="n">username</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">avatar_url</span> <span class="o">=</span> <span class="n">avatar_url</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">tts</span> <span class="o">=</span> <span class="n">tts</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">proxy</span> <span class="o">=</span> <span class="n">proxy</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="kc">None</span>
+
+<div class="viewcode-block" id="DiscordWebhookOperator.execute"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.discord_webhook_operator.DiscordWebhookOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Call the DiscordWebhookHook to post message</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">DiscordWebhookHook</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">http_conn_id</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">webhook_endpoint</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">message</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">avatar_url</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">tts</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">proxy</span>
+ <span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span></div></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/ecs_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/ecs_operator.html b/_modules/airflow/contrib/operators/ecs_operator.html
index 5c181c5..442c5ae 100644
--- a/_modules/airflow/contrib/operators/ecs_operator.html
+++ b/_modules/airflow/contrib/operators/ecs_operator.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,22 @@
<h1>Source code for airflow.contrib.operators.ecs_operator</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="kn">import</span> <span class="nn">sys</span>
<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/emr_add_steps_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/emr_add_steps_operator.html b/_modules/airflow/contrib/operators/emr_add_steps_operator.html
index daac24b..0b0728c 100644
--- a/_modules/airflow/contrib/operators/emr_add_steps_operator.html
+++ b/_modules/airflow/contrib/operators/emr_add_steps_operator.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,22 @@
<h1>Source code for airflow.contrib.operators.emr_add_steps_operator</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
<span class="kn">from</span> <span class="nn">airflow.utils</span> <span class="k">import</span> <span class="n">apply_defaults</span>
<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
@@ -197,7 +204,7 @@
<span class="sd"> :param steps: boto3 style steps to be added to the jobflow</span>
<span class="sd"> :type steps: list</span>
<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'job_flow_id'</span><span class="p">]</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'job_flow_id'</span><span class="p">,</span> <span class="s1">'steps'</span><span class="p">]</span>
<span class="n">template_ext</span> <span class="o">=</span> <span class="p">()</span>
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f9c915'</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/emr_create_job_flow_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/emr_create_job_flow_operator.html b/_modules/airflow/contrib/operators/emr_create_job_flow_operator.html
index 1e477c5..d17d669 100644
--- a/_modules/airflow/contrib/operators/emr_create_job_flow_operator.html
+++ b/_modules/airflow/contrib/operators/emr_create_job_flow_operator.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,22 @@
<h1>Source code for airflow.contrib.operators.emr_create_job_flow_operator</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.emr_hook</span> <span class="k">import</span> <span class="n">EmrHook</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
<span class="kn">from</span> <span class="nn">airflow.utils</span> <span class="k">import</span> <span class="n">apply_defaults</span>
@@ -198,7 +205,7 @@
<span class="sd"> :param job_flow_overrides: boto3 style arguments to override emr_connection extra</span>
<span class="sd"> :type steps: dict</span>
<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'job_flow_overrides'</span><span class="p">]</span>
<span class="n">template_ext</span> <span class="o">=</span> <span class="p">()</span>
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f9c915'</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/emr_terminate_job_flow_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/emr_terminate_job_flow_operator.html b/_modules/airflow/contrib/operators/emr_terminate_job_flow_operator.html
index 7f2f1ca..fdb5053 100644
--- a/_modules/airflow/contrib/operators/emr_terminate_job_flow_operator.html
+++ b/_modules/airflow/contrib/operators/emr_terminate_job_flow_operator.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,22 @@
<h1>Source code for airflow.contrib.operators.emr_terminate_job_flow_operator</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
<span class="kn">from</span> <span class="nn">airflow.utils</span> <span class="k">import</span> <span class="n">apply_defaults</span>
<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
[03/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/sensors/named_hive_partition_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/sensors/named_hive_partition_sensor.html b/_modules/airflow/sensors/named_hive_partition_sensor.html
new file mode 100644
index 0000000..b199984
--- /dev/null
+++ b/_modules/airflow/sensors/named_hive_partition_sensor.html
@@ -0,0 +1,339 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.sensors.named_hive_partition_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.sensors.named_hive_partition_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.sensors.named_hive_partition_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="k">import</span> <span class="n">basestring</span>
+
+<span class="kn">from</span> <span class="nn">airflow.sensors.base_sensor_operator</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="NamedHivePartitionSensor"><a class="viewcode-back" href="../../../code.html#airflow.sensors.named_hive_partition_sensor.NamedHivePartitionSensor">[docs]</a><span class="k">class</span> <span class="nc">NamedHivePartitionSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Waits for a set of partitions to show up in Hive.</span>
+
+<span class="sd"> :param partition_names: List of fully qualified names of the</span>
+<span class="sd"> partitions to wait for. A fully qualified name is of the</span>
+<span class="sd"> form ``schema.table/pk1=pv1/pk2=pv2``, for example,</span>
+<span class="sd"> default.users/ds=2016-01-01. This is passed as is to the metastore</span>
+<span class="sd"> Thrift client ``get_partitions_by_name`` method. Note that</span>
+<span class="sd"> you cannot use logical or comparison operators as in</span>
+<span class="sd"> HivePartitionSensor.</span>
+<span class="sd"> :type partition_names: list of strings</span>
+<span class="sd"> :param metastore_conn_id: reference to the metastore thrift service</span>
+<span class="sd"> connection id</span>
+<span class="sd"> :type metastore_conn_id: str</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'partition_names'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#8d99ae'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">partition_names</span><span class="p">,</span>
+ <span class="n">metastore_conn_id</span><span class="o">=</span><span class="s1">'metastore_default'</span><span class="p">,</span>
+ <span class="n">poke_interval</span><span class="o">=</span><span class="mi">60</span> <span class="o">*</span> <span class="mi">3</span><span class="p">,</span>
+ <span class="n">hook</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">NamedHivePartitionSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+ <span class="n">poke_interval</span><span class="o">=</span><span class="n">poke_interval</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">partition_names</span><span class="p">,</span> <span class="n">basestring</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">'partition_names must be an array of strings'</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span> <span class="o">=</span> <span class="n">metastore_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span> <span class="o">=</span> <span class="n">partition_names</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">hook</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="ow">and</span> <span class="n">metastore_conn_id</span> <span class="o">!=</span> <span class="s1">'metastore_default'</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s1">'A hook was passed but a non default'</span>
+ <span class="s1">'metastore_conn_id='</span>
+ <span class="s1">'</span><span class="si">{}</span><span class="s1"> was used'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">metastore_conn_id</span><span class="p">))</span>
+
+ <span class="nd">@staticmethod</span>
+ <span class="k">def</span> <span class="nf">parse_partition_name</span><span class="p">(</span><span class="n">partition</span><span class="p">):</span>
+ <span class="n">first_split</span> <span class="o">=</span> <span class="n">partition</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">first_split</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
+ <span class="n">schema</span> <span class="o">=</span> <span class="s1">'default'</span>
+ <span class="n">table_partition</span> <span class="o">=</span> <span class="nb">max</span><span class="p">(</span><span class="n">first_split</span><span class="p">)</span> <span class="c1"># poor man first</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">schema</span><span class="p">,</span> <span class="n">table_partition</span> <span class="o">=</span> <span class="n">first_split</span>
+ <span class="n">second_split</span> <span class="o">=</span> <span class="n">table_partition</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'/'</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">second_split</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'Could not parse '</span> <span class="o">+</span> <span class="n">partition</span> <span class="o">+</span>
+ <span class="s1">'into table, partition'</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">table</span><span class="p">,</span> <span class="n">partition</span> <span class="o">=</span> <span class="n">second_split</span>
+ <span class="k">return</span> <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span>
+
+ <span class="k">def</span> <span class="nf">poke_partition</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">partition</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="p">:</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="k">import</span> <span class="n">HiveMetastoreHook</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">(</span>
+ <span class="n">metastore_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span><span class="p">)</span>
+
+ <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_partition_name</span><span class="p">(</span><span class="n">partition</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'Poking for </span><span class="si">{schema}</span><span class="s1">.</span><span class="si">{table}</span><span class="s1">/</span><span class="si">{partition}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
+ <span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">check_for_named_partition</span><span class="p">(</span>
+ <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span> <span class="o">=</span> <span class="p">[</span>
+ <span class="n">partition_name</span> <span class="k">for</span> <span class="n">partition_name</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">poke_partition</span><span class="p">(</span><span class="n">partition_name</span><span class="p">)</span>
+ <span class="p">]</span>
+ <span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/sensors/s3_key_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/sensors/s3_key_sensor.html b/_modules/airflow/sensors/s3_key_sensor.html
new file mode 100644
index 0000000..831f7d3
--- /dev/null
+++ b/_modules/airflow/sensors/s3_key_sensor.html
@@ -0,0 +1,316 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.sensors.s3_key_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.sensors.s3_key_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.sensors.s3_key_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+
+<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="k">import</span> <span class="n">urlparse</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.sensors.base_sensor_operator</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="S3KeySensor"><a class="viewcode-back" href="../../../code.html#airflow.sensors.s3_key_sensor.S3KeySensor">[docs]</a><span class="k">class</span> <span class="nc">S3KeySensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Waits for a key (a file-like instance on S3) to be present in a S3 bucket.</span>
+<span class="sd"> S3 being a key/value it does not support folders. The path is just a key</span>
+<span class="sd"> a resource.</span>
+
+<span class="sd"> :param bucket_key: The key being waited on. Supports full s3:// style url</span>
+<span class="sd"> or relative path from root level.</span>
+<span class="sd"> :type bucket_key: str</span>
+<span class="sd"> :param bucket_name: Name of the S3 bucket</span>
+<span class="sd"> :type bucket_name: str</span>
+<span class="sd"> :param wildcard_match: whether the bucket_key should be interpreted as a</span>
+<span class="sd"> Unix wildcard pattern</span>
+<span class="sd"> :type wildcard_match: bool</span>
+<span class="sd"> :param aws_conn_id: a reference to the s3 connection</span>
+<span class="sd"> :type aws_conn_id: str</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bucket_key'</span><span class="p">,</span> <span class="s1">'bucket_name'</span><span class="p">)</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">bucket_key</span><span class="p">,</span>
+ <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">wildcard_match</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">aws_conn_id</span><span class="o">=</span><span class="s1">'aws_default'</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">S3KeySensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="c1"># Parse</span>
+ <span class="k">if</span> <span class="n">bucket_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">parsed_url</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">bucket_key</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span> <span class="o">==</span> <span class="s1">''</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Please provide a bucket_name'</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">bucket_name</span> <span class="o">=</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span>
+ <span class="k">if</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">path</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'/'</span><span class="p">:</span>
+ <span class="n">bucket_key</span> <span class="o">=</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">path</span><span class="p">[</span><span class="mi">1</span><span class="p">:]</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">bucket_key</span> <span class="o">=</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">path</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bucket_name</span> <span class="o">=</span> <span class="n">bucket_name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bucket_key</span> <span class="o">=</span> <span class="n">bucket_key</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">wildcard_match</span> <span class="o">=</span> <span class="n">wildcard_match</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span> <span class="o">=</span> <span class="n">aws_conn_id</span>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.S3_hook</span> <span class="k">import</span> <span class="n">S3Hook</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">S3Hook</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span>
+ <span class="n">full_url</span> <span class="o">=</span> <span class="s2">"s3://"</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket_name</span> <span class="o">+</span> <span class="s2">"/"</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket_key</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Poking for key : </span><span class="si">{full_url}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">wildcard_match</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">check_for_wildcard_key</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket_key</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bucket_name</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">check_for_key</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket_key</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket_name</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/sensors/s3_prefix_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/sensors/s3_prefix_sensor.html b/_modules/airflow/sensors/s3_prefix_sensor.html
new file mode 100644
index 0000000..c42e8c0
--- /dev/null
+++ b/_modules/airflow/sensors/s3_prefix_sensor.html
@@ -0,0 +1,303 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.sensors.s3_prefix_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.sensors.s3_prefix_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.sensors.s3_prefix_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.sensors.base_sensor_operator</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="S3PrefixSensor"><a class="viewcode-back" href="../../../code.html#airflow.sensors.s3_prefix_sensor.S3PrefixSensor">[docs]</a><span class="k">class</span> <span class="nc">S3PrefixSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Waits for a prefix to exist. A prefix is the first part of a key,</span>
+<span class="sd"> thus enabling checking of constructs similar to glob airfl* or</span>
+<span class="sd"> SQL LIKE 'airfl%'. There is the possibility to precise a delimiter to</span>
+<span class="sd"> indicate the hierarchy or keys, meaning that the match will stop at that</span>
+<span class="sd"> delimiter. Current code accepts sane delimiters, i.e. characters that</span>
+<span class="sd"> are NOT special characters in the Python regex engine.</span>
+
+<span class="sd"> :param bucket_name: Name of the S3 bucket</span>
+<span class="sd"> :type bucket_name: str</span>
+<span class="sd"> :param prefix: The prefix being waited on. Relative path from bucket root level.</span>
+<span class="sd"> :type prefix: str</span>
+<span class="sd"> :param delimiter: The delimiter intended to show hierarchy.</span>
+<span class="sd"> Defaults to '/'.</span>
+<span class="sd"> :type delimiter: str</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'prefix'</span><span class="p">,</span> <span class="s1">'bucket_name'</span><span class="p">)</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">bucket_name</span><span class="p">,</span>
+ <span class="n">prefix</span><span class="p">,</span>
+ <span class="n">delimiter</span><span class="o">=</span><span class="s1">'/'</span><span class="p">,</span>
+ <span class="n">aws_conn_id</span><span class="o">=</span><span class="s1">'aws_default'</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">S3PrefixSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="c1"># Parse</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bucket_name</span> <span class="o">=</span> <span class="n">bucket_name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">prefix</span> <span class="o">=</span> <span class="n">prefix</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delimiter</span> <span class="o">=</span> <span class="n">delimiter</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">full_url</span> <span class="o">=</span> <span class="s2">"s3://"</span> <span class="o">+</span> <span class="n">bucket_name</span> <span class="o">+</span> <span class="s1">'/'</span> <span class="o">+</span> <span class="n">prefix</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span> <span class="o">=</span> <span class="n">aws_conn_id</span>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Poking for prefix : </span><span class="si">{self.prefix}</span><span class="se">\n</span><span class="s1">'</span>
+ <span class="s1">'in bucket s3://</span><span class="si">{self.bucket_name}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.S3_hook</span> <span class="k">import</span> <span class="n">S3Hook</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">S3Hook</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">check_for_prefix</span><span class="p">(</span>
+ <span class="n">prefix</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">prefix</span><span class="p">,</span>
+ <span class="n">delimiter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delimiter</span><span class="p">,</span>
+ <span class="n">bucket_name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket_name</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/sensors/sql_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/sensors/sql_sensor.html b/_modules/airflow/sensors/sql_sensor.html
new file mode 100644
index 0000000..f751164
--- /dev/null
+++ b/_modules/airflow/sensors/sql_sensor.html
@@ -0,0 +1,293 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.sensors.sql_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.sensors.sql_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.sensors.sql_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.sensors.base_sensor_operator</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="SqlSensor"><a class="viewcode-back" href="../../../code.html#airflow.sensors.sql_sensor.SqlSensor">[docs]</a><span class="k">class</span> <span class="nc">SqlSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Runs a sql statement until a criteria is met. It will keep trying while</span>
+<span class="sd"> sql returns no row, or if the first cell in (0, '0', '').</span>
+
+<span class="sd"> :param conn_id: The connection to run the sensor against</span>
+<span class="sd"> :type conn_id: string</span>
+<span class="sd"> :param sql: The sql to run. To pass, it needs to return at least one cell</span>
+<span class="sd"> that contains a non-zero / empty string value.</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'sql'</span><span class="p">,)</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.hql'</span><span class="p">,</span> <span class="s1">'.sql'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#7c7287'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn_id</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span> <span class="o">=</span> <span class="n">conn_id</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">SqlSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">BaseHook</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Poking: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
+ <span class="n">records</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_records</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">records</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">False</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">if</span> <span class="nb">str</span><span class="p">(</span><span class="n">records</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="mi">0</span><span class="p">])</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'0'</span><span class="p">,</span> <span class="s1">''</span><span class="p">,):</span>
+ <span class="k">return</span> <span class="kc">False</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">True</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/sensors/time_delta_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/sensors/time_delta_sensor.html b/_modules/airflow/sensors/time_delta_sensor.html
new file mode 100644
index 0000000..270d60c
--- /dev/null
+++ b/_modules/airflow/sensors/time_delta_sensor.html
@@ -0,0 +1,281 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.sensors.time_delta_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.sensors.time_delta_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.sensors.time_delta_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.sensors.base_sensor_operator</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils</span> <span class="k">import</span> <span class="n">timezone</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="TimeDeltaSensor"><a class="viewcode-back" href="../../../code.html#airflow.sensors.time_delta_sensor.TimeDeltaSensor">[docs]</a><span class="k">class</span> <span class="nc">TimeDeltaSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Waits for a timedelta after the task's execution_date + schedule_interval.</span>
+<span class="sd"> In Airflow, the daily task stamped with ``execution_date``</span>
+<span class="sd"> 2016-01-01 can only start running on 2016-01-02. The timedelta here</span>
+<span class="sd"> represents the time after the execution period has closed.</span>
+
+<span class="sd"> :param delta: time length to wait after execution_date before succeeding</span>
+<span class="sd"> :type delta: datetime.timedelta</span>
+<span class="sd"> """</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">delta</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">TimeDeltaSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delta</span> <span class="o">=</span> <span class="n">delta</span>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">dag</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">'dag'</span><span class="p">]</span>
+ <span class="n">target_dttm</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">following_schedule</span><span class="p">(</span><span class="n">context</span><span class="p">[</span><span class="s1">'execution_date'</span><span class="p">])</span>
+ <span class="n">target_dttm</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">delta</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Checking if the time (</span><span class="si">%s</span><span class="s1">) has come'</span><span class="p">,</span> <span class="n">target_dttm</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">></span> <span class="n">target_dttm</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/sensors/time_sensor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/sensors/time_sensor.html b/_modules/airflow/sensors/time_sensor.html
new file mode 100644
index 0000000..2b2ab52
--- /dev/null
+++ b/_modules/airflow/sensors/time_sensor.html
@@ -0,0 +1,275 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.sensors.time_sensor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
+ <link rel="up" title="Module code" href="../../index.html"/>
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.sensors.time_sensor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.sensors.time_sensor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.sensors.base_sensor_operator</span> <span class="k">import</span> <span class="n">BaseSensorOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils</span> <span class="k">import</span> <span class="n">timezone</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="TimeSensor"><a class="viewcode-back" href="../../../code.html#airflow.sensors.time_sensor.TimeSensor">[docs]</a><span class="k">class</span> <span class="nc">TimeSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Waits until the specified time of the day.</span>
+
+<span class="sd"> :param target_time: time after which the job succeeds</span>
+<span class="sd"> :type target_time: datetime.time</span>
+<span class="sd"> """</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">target_time</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">TimeSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">target_time</span> <span class="o">=</span> <span class="n">target_time</span>
+
+ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Checking if the time (</span><span class="si">%s</span><span class="s1">) has come'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">target_time</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span><span class="o">.</span><span class="n">time</span><span class="p">()</span> <span class="o">></span> <span class="bp">self</span><span class="o">.</span><span class="n">target_time</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
[26/51] [partial] incubator-airflow-site git commit: 1.10.0
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/pubsub_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/pubsub_operator.html b/_modules/airflow/contrib/operators/pubsub_operator.html
new file mode 100644
index 0000000..99a540f
--- /dev/null
+++ b/_modules/airflow/contrib/operators/pubsub_operator.html
@@ -0,0 +1,669 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.pubsub_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.pubsub_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.pubsub_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_pubsub_hook</span> <span class="k">import</span> <span class="n">PubSubHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="PubSubTopicCreateOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.pubsub_operator.PubSubTopicCreateOperator">[docs]</a><span class="k">class</span> <span class="nc">PubSubTopicCreateOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""Create a PubSub topic.</span>
+
+<span class="sd"> By default, if the topic already exists, this operator will</span>
+<span class="sd"> not cause the DAG to fail. ::</span>
+
+<span class="sd"> with DAG('successful DAG') as dag:</span>
+<span class="sd"> (</span>
+<span class="sd"> dag</span>
+<span class="sd"> >> PubSubTopicCreateOperator(project='my-project',</span>
+<span class="sd"> topic='my_new_topic')</span>
+<span class="sd"> >> PubSubTopicCreateOperator(project='my-project',</span>
+<span class="sd"> topic='my_new_topic')</span>
+<span class="sd"> )</span>
+
+<span class="sd"> The operator can be configured to fail if the topic already exists. ::</span>
+
+<span class="sd"> with DAG('failing DAG') as dag:</span>
+<span class="sd"> (</span>
+<span class="sd"> dag</span>
+<span class="sd"> >> PubSubTopicCreateOperator(project='my-project',</span>
+<span class="sd"> topic='my_new_topic')</span>
+<span class="sd"> >> PubSubTopicCreateOperator(project='my-project',</span>
+<span class="sd"> topic='my_new_topic',</span>
+<span class="sd"> fail_if_exists=True)</span>
+<span class="sd"> )</span>
+
+<span class="sd"> Both ``project`` and ``topic`` are templated so you can use</span>
+<span class="sd"> variables in them.</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'project'</span><span class="p">,</span> <span class="s1">'topic'</span><span class="p">]</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#0273d4'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">project</span><span class="p">,</span>
+ <span class="n">topic</span><span class="p">,</span>
+ <span class="n">fail_if_exists</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> :param project: the GCP project ID where the topic will be created</span>
+<span class="sd"> :type project: string</span>
+<span class="sd"> :param topic: the topic to create. Do not include the</span>
+<span class="sd"> full topic path. In other words, instead of</span>
+<span class="sd"> ``projects/{project}/topics/{topic}``, provide only</span>
+<span class="sd"> ``{topic}``. (templated)</span>
+<span class="sd"> :type topic: string</span>
+<span class="sd"> :param gcp_conn_id: The connection ID to use connecting to</span>
+<span class="sd"> Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request</span>
+<span class="sd"> must have domain-wide delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> """</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">PubSubTopicCreateOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">project</span> <span class="o">=</span> <span class="n">project</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">topic</span> <span class="o">=</span> <span class="n">topic</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">fail_if_exists</span> <span class="o">=</span> <span class="n">fail_if_exists</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">PubSubHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+
+ <span class="n">hook</span><span class="o">.</span><span class="n">create_topic</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">topic</span><span class="p">,</span>
+ <span class="n">fail_if_exists</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">fail_if_exists</span><span class="p">)</span></div>
+
+
+<div class="viewcode-block" id="PubSubSubscriptionCreateOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.pubsub_operator.PubSubSubscriptionCreateOperator">[docs]</a><span class="k">class</span> <span class="nc">PubSubSubscriptionCreateOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""Create a PubSub subscription.</span>
+
+<span class="sd"> By default, the subscription will be created in ``topic_project``. If</span>
+<span class="sd"> ``subscription_project`` is specified and the GCP credentials allow, the</span>
+<span class="sd"> Subscription can be created in a different project from its topic.</span>
+
+<span class="sd"> By default, if the subscription already exists, this operator will</span>
+<span class="sd"> not cause the DAG to fail. However, the topic must exist in the project. ::</span>
+
+<span class="sd"> with DAG('successful DAG') as dag:</span>
+<span class="sd"> (</span>
+<span class="sd"> dag</span>
+<span class="sd"> >> PubSubSubscriptionCreateOperator(</span>
+<span class="sd"> topic_project='my-project', topic='my-topic',</span>
+<span class="sd"> subscription='my-subscription')</span>
+<span class="sd"> >> PubSubSubscriptionCreateOperator(</span>
+<span class="sd"> topic_project='my-project', topic='my-topic',</span>
+<span class="sd"> subscription='my-subscription')</span>
+<span class="sd"> )</span>
+
+<span class="sd"> The operator can be configured to fail if the subscription already exists.</span>
+<span class="sd"> ::</span>
+
+<span class="sd"> with DAG('failing DAG') as dag:</span>
+<span class="sd"> (</span>
+<span class="sd"> dag</span>
+<span class="sd"> >> PubSubSubscriptionCreateOperator(</span>
+<span class="sd"> topic_project='my-project', topic='my-topic',</span>
+<span class="sd"> subscription='my-subscription')</span>
+<span class="sd"> >> PubSubSubscriptionCreateOperator(</span>
+<span class="sd"> topic_project='my-project', topic='my-topic',</span>
+<span class="sd"> subscription='my-subscription', fail_if_exists=True)</span>
+<span class="sd"> )</span>
+
+<span class="sd"> Finally, subscription is not required. If not passed, the operator will</span>
+<span class="sd"> generated a universally unique identifier for the subscription's name. ::</span>
+
+<span class="sd"> with DAG('DAG') as dag:</span>
+<span class="sd"> (</span>
+<span class="sd"> dag >> PubSubSubscriptionCreateOperator(</span>
+<span class="sd"> topic_project='my-project', topic='my-topic')</span>
+<span class="sd"> )</span>
+
+<span class="sd"> ``topic_project``, ``topic``, ``subscription``, and</span>
+<span class="sd"> ``subscription`` are templated so you can use variables in them.</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'topic_project'</span><span class="p">,</span> <span class="s1">'topic'</span><span class="p">,</span> <span class="s1">'subscription'</span><span class="p">,</span>
+ <span class="s1">'subscription_project'</span><span class="p">]</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#0273d4'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">topic_project</span><span class="p">,</span>
+ <span class="n">topic</span><span class="p">,</span>
+ <span class="n">subscription</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">subscription_project</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">ack_deadline_secs</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span>
+ <span class="n">fail_if_exists</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> :param topic_project: the GCP project ID where the topic exists</span>
+<span class="sd"> :type topic_project: string</span>
+<span class="sd"> :param topic: the topic to create. Do not include the</span>
+<span class="sd"> full topic path. In other words, instead of</span>
+<span class="sd"> ``projects/{project}/topics/{topic}``, provide only</span>
+<span class="sd"> ``{topic}``. (templated)</span>
+<span class="sd"> :type topic: string</span>
+<span class="sd"> :param subscription: the Pub/Sub subscription name. If empty, a random</span>
+<span class="sd"> name will be generated using the uuid module</span>
+<span class="sd"> :type subscription: string</span>
+<span class="sd"> :param subscription_project: the GCP project ID where the subscription</span>
+<span class="sd"> will be created. If empty, ``topic_project`` will be used.</span>
+<span class="sd"> :type subscription_project: string</span>
+<span class="sd"> :param ack_deadline_secs: Number of seconds that a subscriber has to</span>
+<span class="sd"> acknowledge each message pulled from the subscription</span>
+<span class="sd"> :type ack_deadline_secs: int</span>
+<span class="sd"> :param gcp_conn_id: The connection ID to use connecting to</span>
+<span class="sd"> Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request</span>
+<span class="sd"> must have domain-wide delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> """</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">PubSubSubscriptionCreateOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">topic_project</span> <span class="o">=</span> <span class="n">topic_project</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">topic</span> <span class="o">=</span> <span class="n">topic</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">subscription</span> <span class="o">=</span> <span class="n">subscription</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">subscription_project</span> <span class="o">=</span> <span class="n">subscription_project</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">ack_deadline_secs</span> <span class="o">=</span> <span class="n">ack_deadline_secs</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">fail_if_exists</span> <span class="o">=</span> <span class="n">fail_if_exists</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">PubSubHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">create_subscription</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">topic_project</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">topic</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">subscription</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">subscription_project</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ack_deadline_secs</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">fail_if_exists</span><span class="p">)</span></div>
+
+
+<div class="viewcode-block" id="PubSubTopicDeleteOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.pubsub_operator.PubSubTopicDeleteOperator">[docs]</a><span class="k">class</span> <span class="nc">PubSubTopicDeleteOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""Delete a PubSub topic.</span>
+
+<span class="sd"> By default, if the topic does not exist, this operator will</span>
+<span class="sd"> not cause the DAG to fail. ::</span>
+
+<span class="sd"> with DAG('successful DAG') as dag:</span>
+<span class="sd"> (</span>
+<span class="sd"> dag</span>
+<span class="sd"> >> PubSubTopicDeleteOperator(project='my-project',</span>
+<span class="sd"> topic='non_existing_topic')</span>
+<span class="sd"> )</span>
+
+<span class="sd"> The operator can be configured to fail if the topic does not exist. ::</span>
+
+<span class="sd"> with DAG('failing DAG') as dag:</span>
+<span class="sd"> (</span>
+<span class="sd"> dag</span>
+<span class="sd"> >> PubSubTopicCreateOperator(project='my-project',</span>
+<span class="sd"> topic='non_existing_topic',</span>
+<span class="sd"> fail_if_not_exists=True)</span>
+<span class="sd"> )</span>
+
+<span class="sd"> Both ``project`` and ``topic`` are templated so you can use</span>
+<span class="sd"> variables in them.</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'project'</span><span class="p">,</span> <span class="s1">'topic'</span><span class="p">]</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#cb4335'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">project</span><span class="p">,</span>
+ <span class="n">topic</span><span class="p">,</span>
+ <span class="n">fail_if_not_exists</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> :param project: the GCP project ID in which to work (templated)</span>
+<span class="sd"> :type project: string</span>
+<span class="sd"> :param topic: the topic to delete. Do not include the</span>
+<span class="sd"> full topic path. In other words, instead of</span>
+<span class="sd"> ``projects/{project}/topics/{topic}``, provide only</span>
+<span class="sd"> ``{topic}``. (templated)</span>
+<span class="sd"> :type topic: string</span>
+<span class="sd"> :param fail_if_not_exists: If True and the topic does not exist, fail</span>
+<span class="sd"> the task</span>
+<span class="sd"> :type fail_if_not_exists: bool</span>
+<span class="sd"> :param gcp_conn_id: The connection ID to use connecting to</span>
+<span class="sd"> Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request</span>
+<span class="sd"> must have domain-wide delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> """</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">PubSubTopicDeleteOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">project</span> <span class="o">=</span> <span class="n">project</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">topic</span> <span class="o">=</span> <span class="n">topic</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">fail_if_not_exists</span> <span class="o">=</span> <span class="n">fail_if_not_exists</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">PubSubHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+
+ <span class="n">hook</span><span class="o">.</span><span class="n">delete_topic</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">topic</span><span class="p">,</span>
+ <span class="n">fail_if_not_exists</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">fail_if_not_exists</span><span class="p">)</span></div>
+
+
+<div class="viewcode-block" id="PubSubSubscriptionDeleteOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.pubsub_operator.PubSubSubscriptionDeleteOperator">[docs]</a><span class="k">class</span> <span class="nc">PubSubSubscriptionDeleteOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""Delete a PubSub subscription.</span>
+
+<span class="sd"> By default, if the subscription does not exist, this operator will</span>
+<span class="sd"> not cause the DAG to fail. ::</span>
+
+<span class="sd"> with DAG('successful DAG') as dag:</span>
+<span class="sd"> (</span>
+<span class="sd"> dag</span>
+<span class="sd"> >> PubSubSubscriptionDeleteOperator(project='my-project',</span>
+<span class="sd"> subscription='non-existing')</span>
+<span class="sd"> )</span>
+
+<span class="sd"> The operator can be configured to fail if the subscription already exists.</span>
+
+<span class="sd"> ::</span>
+
+<span class="sd"> with DAG('failing DAG') as dag:</span>
+<span class="sd"> (</span>
+<span class="sd"> dag</span>
+<span class="sd"> >> PubSubSubscriptionDeleteOperator(</span>
+<span class="sd"> project='my-project', subscription='non-existing',</span>
+<span class="sd"> fail_if_not_exists=True)</span>
+<span class="sd"> )</span>
+
+<span class="sd"> ``project``, and ``subscription`` are templated so you can use</span>
+<span class="sd"> variables in them.</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'project'</span><span class="p">,</span> <span class="s1">'subscription'</span><span class="p">]</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#cb4335'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">project</span><span class="p">,</span>
+ <span class="n">subscription</span><span class="p">,</span>
+ <span class="n">fail_if_not_exists</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> :param project: the GCP project ID in which to work (templated)</span>
+<span class="sd"> :type project: string</span>
+<span class="sd"> :param subscription: the subscription to delete. Do not include the</span>
+<span class="sd"> full subscription path. In other words, instead of</span>
+<span class="sd"> ``projects/{project}/subscription/{subscription}``, provide only</span>
+<span class="sd"> ``{subscription}``. (templated)</span>
+<span class="sd"> :type subscription: string</span>
+<span class="sd"> :param fail_if_not_exists: If True and the subscription does not exist,</span>
+<span class="sd"> fail the task</span>
+<span class="sd"> :type fail_if_not_exists: bool</span>
+<span class="sd"> :param gcp_conn_id: The connection ID to use connecting to</span>
+<span class="sd"> Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request</span>
+<span class="sd"> must have domain-wide delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> """</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">PubSubSubscriptionDeleteOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">project</span> <span class="o">=</span> <span class="n">project</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">subscription</span> <span class="o">=</span> <span class="n">subscription</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">fail_if_not_exists</span> <span class="o">=</span> <span class="n">fail_if_not_exists</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">PubSubHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+
+ <span class="n">hook</span><span class="o">.</span><span class="n">delete_subscription</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">subscription</span><span class="p">,</span>
+ <span class="n">fail_if_not_exists</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">fail_if_not_exists</span><span class="p">)</span></div>
+
+
+<div class="viewcode-block" id="PubSubPublishOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.pubsub_operator.PubSubPublishOperator">[docs]</a><span class="k">class</span> <span class="nc">PubSubPublishOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""Publish messages to a PubSub topic.</span>
+
+<span class="sd"> Each Task publishes all provided messages to the same topic</span>
+<span class="sd"> in a single GCP project. If the topic does not exist, this</span>
+<span class="sd"> task will fail. ::</span>
+
+<span class="sd"> from base64 import b64encode as b64e</span>
+
+<span class="sd"> m1 = {'data': b64e('Hello, World!'),</span>
+<span class="sd"> 'attributes': {'type': 'greeting'}</span>
+<span class="sd"> }</span>
+<span class="sd"> m2 = {'data': b64e('Knock, knock')}</span>
+<span class="sd"> m3 = {'attributes': {'foo': ''}}</span>
+
+<span class="sd"> t1 = PubSubPublishOperator(</span>
+<span class="sd"> project='my-project',topic='my_topic',</span>
+<span class="sd"> messages=[m1, m2, m3],</span>
+<span class="sd"> create_topic=True,</span>
+<span class="sd"> dag=dag)</span>
+
+<span class="sd"> ``project`` , ``topic``, and ``messages`` are templated so you can use</span>
+<span class="sd"> variables in them.</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'project'</span><span class="p">,</span> <span class="s1">'topic'</span><span class="p">,</span> <span class="s1">'messages'</span><span class="p">]</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#0273d4'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">project</span><span class="p">,</span>
+ <span class="n">topic</span><span class="p">,</span>
+ <span class="n">messages</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> :param project: the GCP project ID in which to work (templated)</span>
+<span class="sd"> :type project: string</span>
+<span class="sd"> :param topic: the topic to which to publish. Do not include the</span>
+<span class="sd"> full topic path. In other words, instead of</span>
+<span class="sd"> ``projects/{project}/topics/{topic}``, provide only</span>
+<span class="sd"> ``{topic}``. (templated)</span>
+<span class="sd"> :type topic: string</span>
+<span class="sd"> :param messages: a list of messages to be published to the</span>
+<span class="sd"> topic. Each message is a dict with one or more of the</span>
+<span class="sd"> following keys-value mappings:</span>
+<span class="sd"> * 'data': a base64-encoded string</span>
+<span class="sd"> * 'attributes': {'key1': 'value1', ...}</span>
+<span class="sd"> Each message must contain at least a non-empty 'data' value</span>
+<span class="sd"> or an attribute dict with at least one key. See</span>
+<span class="sd"> https://cloud.google.com/pubsub/docs/reference/rest/v1/PubsubMessage</span>
+<span class="sd"> (templated)</span>
+<span class="sd"> :type messages: list</span>
+<span class="sd"> :param gcp_conn_id: The connection ID to use connecting to</span>
+<span class="sd"> Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request</span>
+<span class="sd"> must have domain-wide delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> """</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">PubSubPublishOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">project</span> <span class="o">=</span> <span class="n">project</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">topic</span> <span class="o">=</span> <span class="n">topic</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">messages</span> <span class="o">=</span> <span class="n">messages</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">PubSubHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+ <span class="n">hook</span><span class="o">.</span><span class="n">publish</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">topic</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">messages</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/qubole_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/qubole_operator.html b/_modules/airflow/contrib/operators/qubole_operator.html
new file mode 100644
index 0000000..b14626b
--- /dev/null
+++ b/_modules/airflow/contrib/operators/qubole_operator.html
@@ -0,0 +1,411 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.qubole_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.qubole_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.qubole_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.qubole_hook</span> <span class="k">import</span> <span class="n">QuboleHook</span>
+
+
+<div class="viewcode-block" id="QuboleOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.qubole_operator.QuboleOperator">[docs]</a><span class="k">class</span> <span class="nc">QuboleOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Execute tasks (commands) on QDS (https://qubole.com).</span>
+
+<span class="sd"> :param qubole_conn_id: Connection id which consists of qds auth_token</span>
+<span class="sd"> :type qubole_conn_id: str</span>
+
+<span class="sd"> kwargs:</span>
+<span class="sd"> :command_type: type of command to be executed, e.g. hivecmd, shellcmd, hadoopcmd</span>
+<span class="sd"> :tags: array of tags to be assigned with the command</span>
+<span class="sd"> :cluster_label: cluster label on which the command will be executed</span>
+<span class="sd"> :name: name to be given to command</span>
+<span class="sd"> :notify: whether to send email on command completion or not (default is False)</span>
+
+<span class="sd"> **Arguments specific to command types**</span>
+
+<span class="sd"> hivecmd:</span>
+<span class="sd"> :query: inline query statement</span>
+<span class="sd"> :script_location: s3 location containing query statement</span>
+<span class="sd"> :sample_size: size of sample in bytes on which to run query</span>
+<span class="sd"> :macros: macro values which were used in query</span>
+<span class="sd"> prestocmd:</span>
+<span class="sd"> :query: inline query statement</span>
+<span class="sd"> :script_location: s3 location containing query statement</span>
+<span class="sd"> :macros: macro values which were used in query</span>
+<span class="sd"> hadoopcmd:</span>
+<span class="sd"> :sub_commnad: must be one these ["jar", "s3distcp", "streaming"] followed by</span>
+<span class="sd"> 1 or more args</span>
+<span class="sd"> shellcmd:</span>
+<span class="sd"> :script: inline command with args</span>
+<span class="sd"> :script_location: s3 location containing query statement</span>
+<span class="sd"> :files: list of files in s3 bucket as file1,file2 format. These files will be</span>
+<span class="sd"> copied into the working directory where the qubole command is being</span>
+<span class="sd"> executed.</span>
+<span class="sd"> :archives: list of archives in s3 bucket as archive1,archive2 format. These</span>
+<span class="sd"> will be unarchived intothe working directory where the qubole command is</span>
+<span class="sd"> being executed</span>
+<span class="sd"> :parameters: any extra args which need to be passed to script (only when</span>
+<span class="sd"> script_location is supplied)</span>
+<span class="sd"> pigcmd:</span>
+<span class="sd"> :script: inline query statement (latin_statements)</span>
+<span class="sd"> :script_location: s3 location containing pig query</span>
+<span class="sd"> :parameters: any extra args which need to be passed to script (only when</span>
+<span class="sd"> script_location is supplied</span>
+<span class="sd"> sparkcmd:</span>
+<span class="sd"> :program: the complete Spark Program in Scala, SQL, Command, R, or Python</span>
+<span class="sd"> :cmdline: spark-submit command line, all required information must be specify</span>
+<span class="sd"> in cmdline itself.</span>
+<span class="sd"> :sql: inline sql query</span>
+<span class="sd"> :script_location: s3 location containing query statement</span>
+<span class="sd"> :language: language of the program, Scala, SQL, Command, R, or Python</span>
+<span class="sd"> :app_id: ID of an Spark job server app</span>
+<span class="sd"> :arguments: spark-submit command line arguments</span>
+<span class="sd"> :user_program_arguments: arguments that the user program takes in</span>
+<span class="sd"> :macros: macro values which were used in query</span>
+<span class="sd"> dbtapquerycmd:</span>
+<span class="sd"> :db_tap_id: data store ID of the target database, in Qubole.</span>
+<span class="sd"> :query: inline query statement</span>
+<span class="sd"> :macros: macro values which were used in query</span>
+<span class="sd"> dbexportcmd:</span>
+<span class="sd"> :mode: 1 (simple), 2 (advance)</span>
+<span class="sd"> :hive_table: Name of the hive table</span>
+<span class="sd"> :partition_spec: partition specification for Hive table.</span>
+<span class="sd"> :dbtap_id: data store ID of the target database, in Qubole.</span>
+<span class="sd"> :db_table: name of the db table</span>
+<span class="sd"> :db_update_mode: allowinsert or updateonly</span>
+<span class="sd"> :db_update_keys: columns used to determine the uniqueness of rows</span>
+<span class="sd"> :export_dir: HDFS/S3 location from which data will be exported.</span>
+<span class="sd"> :fields_terminated_by: hex of the char used as column separator in the dataset</span>
+<span class="sd"> dbimportcmd:</span>
+<span class="sd"> :mode: 1 (simple), 2 (advance)</span>
+<span class="sd"> :hive_table: Name of the hive table</span>
+<span class="sd"> :dbtap_id: data store ID of the target database, in Qubole.</span>
+<span class="sd"> :db_table: name of the db table</span>
+<span class="sd"> :where_clause: where clause, if any</span>
+<span class="sd"> :parallelism: number of parallel db connections to use for extracting data</span>
+<span class="sd"> :extract_query: SQL query to extract data from db. $CONDITIONS must be part</span>
+<span class="sd"> of the where clause.</span>
+<span class="sd"> :boundary_query: Query to be used get range of row IDs to be extracted</span>
+<span class="sd"> :split_column: Column used as row ID to split data into ranges (mode 2)</span>
+
+<span class="sd"> .. note:: Following fields are template-supported : ``query``, ``script_location``,</span>
+<span class="sd"> ``sub_command``, ``script``, ``files``, ``archives``, ``program``, ``cmdline``,</span>
+<span class="sd"> ``sql``, ``where_clause``, ``extract_query``, ``boundary_query``, ``macros``,</span>
+<span class="sd"> ``tags``, ``name``, ``parameters``, ``dbtap_id``, ``hive_table``, ``db_table``,</span>
+<span class="sd"> ``split_column``, ``note_id``, ``db_update_keys``, ``export_dir``,</span>
+<span class="sd"> ``partition_spec``, ``qubole_conn_id``, ``arguments``, ``user_program_arguments``.</span>
+<span class="sd"> You can also use ``.txt`` files for template driven use cases.</span>
+
+<span class="sd"> .. note:: In QuboleOperator there is a default handler for task failures and retries,</span>
+<span class="sd"> which generally kills the command running at QDS for the corresponding task</span>
+<span class="sd"> instance. You can override this behavior by providing your own failure and retry</span>
+<span class="sd"> handler in task definition.</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'query'</span><span class="p">,</span> <span class="s1">'script_location'</span><span class="p">,</span> <span class="s1">'sub_command'</span><span class="p">,</span> <span class="s1">'script'</span><span class="p">,</span> <span class="s1">'files'</span><span class="p">,</span>
+ <span class="s1">'archives'</span><span class="p">,</span> <span class="s1">'program'</span><span class="p">,</span> <span class="s1">'cmdline'</span><span class="p">,</span> <span class="s1">'sql'</span><span class="p">,</span> <span class="s1">'where_clause'</span><span class="p">,</span> <span class="s1">'tags'</span><span class="p">,</span>
+ <span class="s1">'extract_query'</span><span class="p">,</span> <span class="s1">'boundary_query'</span><span class="p">,</span> <span class="s1">'macros'</span><span class="p">,</span> <span class="s1">'name'</span><span class="p">,</span> <span class="s1">'parameters'</span><span class="p">,</span>
+ <span class="s1">'dbtap_id'</span><span class="p">,</span> <span class="s1">'hive_table'</span><span class="p">,</span> <span class="s1">'db_table'</span><span class="p">,</span> <span class="s1">'split_column'</span><span class="p">,</span> <span class="s1">'note_id'</span><span class="p">,</span>
+ <span class="s1">'db_update_keys'</span><span class="p">,</span> <span class="s1">'export_dir'</span><span class="p">,</span> <span class="s1">'partition_spec'</span><span class="p">,</span> <span class="s1">'qubole_conn_id'</span><span class="p">,</span>
+ <span class="s1">'arguments'</span><span class="p">,</span> <span class="s1">'user_program_arguments'</span><span class="p">)</span>
+
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.txt'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#3064A1'</span>
+ <span class="n">ui_fgcolor</span> <span class="o">=</span> <span class="s1">'#fff'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">qubole_conn_id</span><span class="o">=</span><span class="s2">"qubole_default"</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">args</span> <span class="o">=</span> <span class="n">args</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span> <span class="o">=</span> <span class="n">kwargs</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">[</span><span class="s1">'qubole_conn_id'</span><span class="p">]</span> <span class="o">=</span> <span class="n">qubole_conn_id</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">QuboleOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">on_failure_callback</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">on_failure_callback</span> <span class="o">=</span> <span class="n">QuboleHook</span><span class="o">.</span><span class="n">handle_failure_retry</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">on_retry_callback</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">on_retry_callback</span> <span class="o">=</span> <span class="n">QuboleHook</span><span class="o">.</span><span class="n">handle_failure_retry</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">context</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ti</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span><span class="o">.</span><span class="n">kill</span><span class="p">(</span><span class="n">ti</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">get_results</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ti</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">fp</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">inline</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">delim</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">fetch</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span><span class="o">.</span><span class="n">get_results</span><span class="p">(</span><span class="n">ti</span><span class="p">,</span> <span class="n">fp</span><span class="p">,</span> <span class="n">inline</span><span class="p">,</span> <span class="n">delim</span><span class="p">,</span> <span class="n">fetch</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">get_log</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ti</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span><span class="o">.</span><span class="n">get_log</span><span class="p">(</span><span class="n">ti</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">get_jobs_id</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ti</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span><span class="o">.</span><span class="n">get_jobs_id</span><span class="p">(</span><span class="n">ti</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">get_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="c1"># Reinitiating the hook, as some template fields might have changed</span>
+ <span class="k">return</span> <span class="n">QuboleHook</span><span class="p">(</span><span class="o">*</span><span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">__getattribute__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">QuboleOperator</span><span class="o">.</span><span class="n">template_fields</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">name</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">:</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">[</span><span class="n">name</span><span class="p">]</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">return</span> <span class="s1">''</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">return</span> <span class="nb">object</span><span class="o">.</span><span class="fm">__getattribute__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">__setattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">QuboleOperator</span><span class="o">.</span><span class="n">template_fields</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="n">value</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="nb">object</span><span class="o">.</span><span class="fm">__setattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/operators/s3_list_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/s3_list_operator.html b/_modules/airflow/contrib/operators/s3_list_operator.html
new file mode 100644
index 0000000..531f835
--- /dev/null
+++ b/_modules/airflow/contrib/operators/s3_list_operator.html
@@ -0,0 +1,317 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.s3_list_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+
+
+
+
+
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
+ <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+ <link rel="up" title="Module code" href="../../../index.html"/>
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.s3_list_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.s3_list_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.S3_hook</span> <span class="k">import</span> <span class="n">S3Hook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="S3ListOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.s3_list_operator.S3ListOperator">[docs]</a><span class="k">class</span> <span class="nc">S3ListOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> List all objects from the bucket with the given string prefix in name.</span>
+
+<span class="sd"> This operator returns a python list with the name of objects which can be</span>
+<span class="sd"> used by `xcom` in the downstream task.</span>
+
+<span class="sd"> :param bucket: The S3 bucket where to find the objects.</span>
+<span class="sd"> :type bucket: string</span>
+<span class="sd"> :param prefix: Prefix string to filters the objects whose name begin with</span>
+<span class="sd"> such prefix</span>
+<span class="sd"> :type prefix: string</span>
+<span class="sd"> :param delimiter: the delimiter marks key hierarchy.</span>
+<span class="sd"> :type delimiter: string</span>
+<span class="sd"> :param aws_conn_id: The connection ID to use when connecting to S3 storage.</span>
+<span class="sd"> :type aws_conn_id: string</span>
+
+<span class="sd"> **Example**:</span>
+<span class="sd"> The following operator would list all the files</span>
+<span class="sd"> (excluding subfolders) from the S3</span>
+<span class="sd"> ``customers/2018/04/`` key in the ``data`` bucket. ::</span>
+
+<span class="sd"> s3_file = S3ListOperator(</span>
+<span class="sd"> task_id='list_3s_files',</span>
+<span class="sd"> bucket='data',</span>
+<span class="sd"> prefix='customers/2018/04/',</span>
+<span class="sd"> delimiter='/',</span>
+<span class="sd"> aws_conn_id='aws_customers_conn'</span>
+<span class="sd"> )</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bucket'</span><span class="p">,</span> <span class="s1">'prefix'</span><span class="p">,</span> <span class="s1">'delimiter'</span><span class="p">)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#ffd700'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">bucket</span><span class="p">,</span>
+ <span class="n">prefix</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span>
+ <span class="n">delimiter</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span>
+ <span class="n">aws_conn_id</span><span class="o">=</span><span class="s1">'aws_default'</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">S3ListOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">prefix</span> <span class="o">=</span> <span class="n">prefix</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delimiter</span> <span class="o">=</span> <span class="n">delimiter</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span> <span class="o">=</span> <span class="n">aws_conn_id</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">S3Hook</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'Getting the list of files from bucket: </span><span class="si">{0}</span><span class="s1"> in prefix: </span><span class="si">{1}</span><span class="s1"> (Delimiter </span><span class="si">{2}</span><span class="s1">)'</span><span class="o">.</span>
+ <span class="nb">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">prefix</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">delimiter</span><span class="p">))</span>
+
+ <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">list_keys</span><span class="p">(</span>
+ <span class="n">bucket_name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span>
+ <span class="n">prefix</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">prefix</span><span class="p">,</span>
+ <span class="n">delimiter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delimiter</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+ <div class="articleComments">
+
+ </div>
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+
+
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.StickyNav.enable();
+ });
+ </script>
+
+
+</body>
+</html>
\ No newline at end of file