You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ma...@apache.org on 2016/08/11 22:59:44 UTC
[5/7] incubator-airflow-site git commit: New version of the docs
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/airflow/operators/sensors.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/sensors.html b/_modules/airflow/operators/sensors.html
index 65a0587..3b716d3 100644
--- a/_modules/airflow/operators/sensors.html
+++ b/_modules/airflow/operators/sensors.html
@@ -155,6 +155,8 @@
<span class="kn">from</span> <span class="nn">future</span> <span class="kn">import</span> <span class="n">standard_library</span>
<span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
+<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="kn">import</span> <span class="nb">basestring</span>
+
<span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span>
<span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urlparse</span>
@@ -229,6 +231,7 @@
<span class="sd"> """</span>
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'sql'</span><span class="p">,)</span>
<span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.hql'</span><span class="p">,</span> <span class="s1">'.sql'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#7c7287'</span>
<span class="nd">@apply_defaults</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn_id</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
@@ -272,6 +275,7 @@
<span class="sd"> :type mysql_conn_id: str</span>
<span class="sd"> """</span>
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'partition_name'</span><span class="p">,</span> <span class="s1">'table'</span><span class="p">,</span> <span class="s1">'schema'</span><span class="p">)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#8da7be'</span>
<span class="nd">@apply_defaults</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
@@ -327,6 +331,7 @@
<span class="sd"> or execution_date_fn can be passed to ExternalTaskSensor, but not both.</span>
<span class="sd"> :type execution_date_fn: callable</span>
<span class="sd"> """</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#19647e'</span>
<span class="nd">@apply_defaults</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
@@ -382,10 +387,11 @@
<span class="sd"> :param partition_names: List of fully qualified names of the</span>
<span class="sd"> partitions to wait for. A fully qualified name is of the</span>
-<span class="sd"> form schema.table/pk1=pv1/pk2=pv2, for example,</span>
+<span class="sd"> form ``schema.table/pk1=pv1/pk2=pv2``, for example,</span>
<span class="sd"> default.users/ds=2016-01-01. This is passed as is to the metastore</span>
-<span class="sd"> Thrift client "get_partitions_by_name" method. Note that</span>
-<span class="sd"> you cannot use logical operators as in HivePartitionSensor.</span>
+<span class="sd"> Thrift client ``get_partitions_by_name`` method. Note that</span>
+<span class="sd"> you cannot use logical or comparison operators as in</span>
+<span class="sd"> HivePartitionSensor.</span>
<span class="sd"> :type partition_names: list of strings</span>
<span class="sd"> :param metastore_conn_id: reference to the metastore thrift service</span>
<span class="sd"> connection id</span>
@@ -393,6 +399,7 @@
<span class="sd"> """</span>
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'partition_names'</span><span class="p">,</span> <span class="p">)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#8d99ae'</span>
<span class="nd">@apply_defaults</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
@@ -408,9 +415,6 @@
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">partition_names</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">'partition_names must be an array of strings'</span><span class="p">)</span>
- <span class="k">for</span> <span class="n">partition_name</span> <span class="ow">in</span> <span class="n">partition_names</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">parse_partition_name</span><span class="p">(</span><span class="n">partition_name</span><span class="p">)</span>
-
<span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span> <span class="o">=</span> <span class="n">metastore_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span> <span class="o">=</span> <span class="n">partition_names</span>
<span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o">=</span> <span class="mi">0</span>
@@ -452,7 +456,7 @@
<span class="sd">"""</span>
<span class="sd"> Waits for a partition to show up in Hive.</span>
-<span class="sd"> Note: Because @partition supports general logical operators, it</span>
+<span class="sd"> Note: Because ``partition`` supports general logical operators, it</span>
<span class="sd"> can be inefficient. Consider using NamedHivePartitionSensor instead if</span>
<span class="sd"> you don't need the full flexibility of HivePartitionSensor.</span>
@@ -460,15 +464,16 @@
<span class="sd"> notation (my_database.my_table)</span>
<span class="sd"> :type table: string</span>
<span class="sd"> :param partition: The partition clause to wait for. This is passed as</span>
-<span class="sd"> is to the metastore Thrift client "get_partitions_by_filter" method,</span>
-<span class="sd"> and apparently supports SQL like notation as in `ds='2015-01-01'</span>
-<span class="sd"> AND type='value'` and > < sings as in "ds>=2015-01-01"</span>
+<span class="sd"> is to the metastore Thrift client ``get_partitions_by_filter`` method,</span>
+<span class="sd"> and apparently supports SQL like notation as in ``ds='2015-01-01'</span>
+<span class="sd"> AND type='value'`` and comparison operators as in ``"ds>=2015-01-01"``</span>
<span class="sd"> :type partition: string</span>
<span class="sd"> :param metastore_conn_id: reference to the metastore thrift service</span>
<span class="sd"> connection id</span>
<span class="sd"> :type metastore_conn_id: str</span>
<span class="sd"> """</span>
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'schema'</span><span class="p">,</span> <span class="s1">'table'</span><span class="p">,</span> <span class="s1">'partition'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#2b2d42'</span>
<span class="nd">@apply_defaults</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
@@ -505,6 +510,7 @@
<span class="sd"> Waits for a file or folder to land in HDFS</span>
<span class="sd"> """</span>
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'filepath'</span><span class="p">,)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#4d9de0'</span>
<span class="nd">@apply_defaults</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/bash_operator.html
----------------------------------------------------------------------
diff --git a/_modules/bash_operator.html b/_modules/bash_operator.html
index d38a3f8..9877b4a 100644
--- a/_modules/bash_operator.html
+++ b/_modules/bash_operator.html
@@ -170,6 +170,9 @@
<span class="sd"> :param bash_command: The command, set of commands or reference to a</span>
<span class="sd"> bash script (must be '.sh') to be executed.</span>
<span class="sd"> :type bash_command: string</span>
+<span class="sd"> :param xcom_push: If xcom_push is True, the last line written to stdout</span>
+<span class="sd"> will also be pushed to an XCom when the bash command completes.</span>
+<span class="sd"> :type xcom_push: bool</span>
<span class="sd"> :param env: If env is not None, it must be a mapping that defines the</span>
<span class="sd"> environment variables for the new process; these are used instead</span>
<span class="sd"> of inheriting the current process environment, which is the default</span>
@@ -189,10 +192,7 @@
<span class="n">env</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
<span class="n">output_encoding</span><span class="o">=</span><span class="s1">'utf-8'</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> If xcom_push is True, the last line written to stdout will also</span>
-<span class="sd"> be pushed to an XCom when the bash command completes.</span>
-<span class="sd"> """</span>
+
<span class="nb">super</span><span class="p">(</span><span class="n">BashOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bash_command</span> <span class="o">=</span> <span class="n">bash_command</span>
<span class="bp">self</span><span class="o">.</span><span class="n">env</span> <span class="o">=</span> <span class="n">env</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/bigquery_hook.html
----------------------------------------------------------------------
diff --git a/_modules/bigquery_hook.html b/_modules/bigquery_hook.html
index 9478587..1f8170a 100644
--- a/_modules/bigquery_hook.html
+++ b/_modules/bigquery_hook.html
@@ -157,6 +157,9 @@
<span class="sd">implementation for BigQuery.</span>
<span class="sd">"""</span>
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">range</span>
+<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="kn">import</span> <span class="nb">basestring</span>
+
<span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">import</span> <span class="nn">time</span>
@@ -440,10 +443,10 @@
<span class="bp">self</span><span class="o">.</span><span class="n">_split_project_dataset_table_input</span><span class="p">(</span>
<span class="s1">'source_project_dataset_table'</span><span class="p">,</span> <span class="n">source_project_dataset_table</span><span class="p">)</span>
<span class="n">source_project_dataset_tables_fixup</span><span class="o">.</span><span class="n">append</span><span class="p">({</span>
- <span class="s1">'projectId'</span><span class="p">:</span> <span class="n">source_project</span><span class="p">,</span>
- <span class="s1">'datasetId'</span><span class="p">:</span> <span class="n">source_dataset</span><span class="p">,</span>
- <span class="s1">'tableId'</span><span class="p">:</span> <span class="n">source_table</span>
- <span class="p">})</span>
+ <span class="s1">'projectId'</span><span class="p">:</span> <span class="n">source_project</span><span class="p">,</span>
+ <span class="s1">'datasetId'</span><span class="p">:</span> <span class="n">source_dataset</span><span class="p">,</span>
+ <span class="s1">'tableId'</span><span class="p">:</span> <span class="n">source_table</span>
+ <span class="p">})</span>
<span class="k">assert</span> <span class="mi">3</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">destination_project_dataset_table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)),</span> <span class="p">(</span>
<span class="s1">'Expected destination_project_dataset_table in the format of '</span>
@@ -652,29 +655,34 @@
<span class="sd"> """</span>
<span class="c1"># check to see if the table exists</span>
<span class="n">table_id</span> <span class="o">=</span> <span class="n">table_resource</span><span class="p">[</span><span class="s1">'tableReference'</span><span class="p">][</span><span class="s1">'tableId'</span><span class="p">]</span>
- <span class="n">table_exists</span> <span class="o">=</span> <span class="bp">False</span>
<span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span> <span class="k">if</span> <span class="n">project_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
<span class="n">tables_list_resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">list</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
<span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
- <span class="k">if</span> <span class="s1">'tables'</span> <span class="ow">in</span> <span class="n">tables_list_resp</span><span class="p">:</span>
- <span class="k">for</span> <span class="n">table</span> <span class="ow">in</span> <span class="n">tables_list_resp</span><span class="p">[</span><span class="s1">'tables'</span><span class="p">]:</span>
+ <span class="k">while</span> <span class="bp">True</span><span class="p">:</span>
+ <span class="k">for</span> <span class="n">table</span> <span class="ow">in</span> <span class="n">tables_list_resp</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'tables'</span><span class="p">,</span> <span class="p">[]):</span>
<span class="k">if</span> <span class="n">table</span><span class="p">[</span><span class="s1">'tableReference'</span><span class="p">][</span><span class="s1">'tableId'</span><span class="p">]</span> <span class="o">==</span> <span class="n">table_id</span><span class="p">:</span>
- <span class="n">table_exists</span> <span class="o">=</span> <span class="bp">True</span>
- <span class="k">break</span>
-
- <span class="c1"># do update if table exists</span>
- <span class="k">if</span> <span class="n">table_exists</span><span class="p">:</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> exists, updating.'</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
- <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
- <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span><span class="p">,</span>
- <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
- <span class="c1"># do insert if table does not exist</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> does not exist. creating.'</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
- <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
- <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="c1"># found the table, do update</span>
+ <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> exists, updating.'</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+ <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span><span class="p">,</span>
+ <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="c1"># If there is a next page, we need to check the next page.</span>
+ <span class="k">if</span> <span class="s1">'nextPageToken'</span> <span class="ow">in</span> <span class="n">tables_list_resp</span><span class="p">:</span>
+ <span class="n">tables_list_resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span>\
+ <span class="o">.</span><span class="n">list</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+ <span class="n">pageToken</span><span class="o">=</span><span class="n">tables_list_resp</span><span class="p">[</span><span class="s1">'nextPageToken'</span><span class="p">])</span>\
+ <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="c1"># If there is no next page, then the table doesn't exist.</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="c1"># do insert</span>
+ <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> does not exist. creating.'</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+ <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">run_grant_dataset_view_access</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
<span class="n">source_dataset</span><span class="p">,</span>
@@ -851,7 +859,7 @@
<span class="k">if</span> <span class="n">size</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="n">size</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">arraysize</span>
<span class="n">result</span> <span class="o">=</span> <span class="p">[]</span>
- <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">xrange</span><span class="p">(</span><span class="n">size</span><span class="p">):</span>
+ <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">size</span><span class="p">):</span>
<span class="n">one</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span>
<span class="k">if</span> <span class="n">one</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="k">break</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/cloudant_hook.html
----------------------------------------------------------------------
diff --git a/_modules/cloudant_hook.html b/_modules/cloudant_hook.html
index b238080..b7747b4 100644
--- a/_modules/cloudant_hook.html
+++ b/_modules/cloudant_hook.html
@@ -150,9 +150,11 @@
<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
<span class="c1"># See the License for the specific language governing permissions and</span>
<span class="c1"># limitations under the License.</span>
-<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="kn">import</span> <span class="nb">unicode</span>
<span class="kn">import</span> <span class="nn">cloudant</span>
+<span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="kn">import</span> <span class="n">BaseHook</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/dagrun_operator.html
----------------------------------------------------------------------
diff --git a/_modules/dagrun_operator.html b/_modules/dagrun_operator.html
index c6725bb..1d147e0 100644
--- a/_modules/dagrun_operator.html
+++ b/_modules/dagrun_operator.html
@@ -185,6 +185,7 @@
<span class="n">template_fields</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">()</span>
<span class="n">template_ext</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">()</span>
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#ffefeb'</span>
+
<span class="nd">@apply_defaults</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/dbapi_hook.html
----------------------------------------------------------------------
diff --git a/_modules/dbapi_hook.html b/_modules/dbapi_hook.html
index e608a23..604dddd 100644
--- a/_modules/dbapi_hook.html
+++ b/_modules/dbapi_hook.html
@@ -195,9 +195,8 @@
<span class="n">username</span><span class="o">=</span><span class="n">db</span><span class="o">.</span><span class="n">login</span><span class="p">,</span>
<span class="n">schema</span><span class="o">=</span><span class="n">db</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span></div>
-
<div class="viewcode-block" id="DbApiHook.get_pandas_df"><a class="viewcode-back" href="../code.html#airflow.hooks.DbApiHook.get_pandas_df">[docs]</a> <span class="k">def</span> <span class="nf">get_pandas_df</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
- <span class="sd">'''</span>
+ <span class="sd">"""</span>
<span class="sd"> Executes the sql and returns a pandas dataframe</span>
<span class="sd"> :param sql: the sql statement to be executed (str) or a list of</span>
@@ -205,7 +204,7 @@
<span class="sd"> :type sql: str or list</span>
<span class="sd"> :param parameters: The parameters to render the SQL query with.</span>
<span class="sd"> :type parameters: mapping or iterable</span>
-<span class="sd"> '''</span>
+<span class="sd"> """</span>
<span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o"><</span> <span class="mi">3</span><span class="p">:</span>
<span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span>
<span class="kn">import</span> <span class="nn">pandas.io.sql</span> <span class="kn">as</span> <span class="nn">psql</span>
@@ -215,7 +214,7 @@
<span class="k">return</span> <span class="n">df</span></div>
<div class="viewcode-block" id="DbApiHook.get_records"><a class="viewcode-back" href="../code.html#airflow.hooks.DbApiHook.get_records">[docs]</a> <span class="k">def</span> <span class="nf">get_records</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
- <span class="sd">'''</span>
+ <span class="sd">"""</span>
<span class="sd"> Executes the sql and returns a set of records.</span>
<span class="sd"> :param sql: the sql statement to be executed (str) or a list of</span>
@@ -223,7 +222,7 @@
<span class="sd"> :type sql: str or list</span>
<span class="sd"> :param parameters: The parameters to render the SQL query with.</span>
<span class="sd"> :type parameters: mapping or iterable</span>
-<span class="sd"> '''</span>
+<span class="sd"> """</span>
<span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o"><</span> <span class="mi">3</span><span class="p">:</span>
<span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span>
<span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
@@ -238,7 +237,7 @@
<span class="k">return</span> <span class="n">rows</span></div>
<div class="viewcode-block" id="DbApiHook.get_first"><a class="viewcode-back" href="../code.html#airflow.hooks.DbApiHook.get_first">[docs]</a> <span class="k">def</span> <span class="nf">get_first</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
- <span class="sd">'''</span>
+ <span class="sd">"""</span>
<span class="sd"> Executes the sql and returns the first resulting row.</span>
<span class="sd"> :param sql: the sql statement to be executed (str) or a list of</span>
@@ -246,7 +245,7 @@
<span class="sd"> :type sql: str or list</span>
<span class="sd"> :param parameters: The parameters to render the SQL query with.</span>
<span class="sd"> :type parameters: mapping or iterable</span>
-<span class="sd"> '''</span>
+<span class="sd"> """</span>
<span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o"><</span> <span class="mi">3</span><span class="p">:</span>
<span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span>
<span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
@@ -280,7 +279,7 @@
<span class="n">sql</span> <span class="o">=</span> <span class="p">[</span><span class="n">sql</span><span class="p">]</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">supports_autocommit</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">set_autocommit</span><span class="p">(</span><span class="n">conn</span><span class="p">,</span> <span class="n">autocommit</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">set_autocommit</span><span class="p">(</span><span class="n">conn</span><span class="p">,</span> <span class="n">autocommit</span><span class="p">)</span>
<span class="n">cur</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
<span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">sql</span><span class="p">:</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/druid_hook.html
----------------------------------------------------------------------
diff --git a/_modules/druid_hook.html b/_modules/druid_hook.html
index f5b2a4d..20ffb1d 100644
--- a/_modules/druid_hook.html
+++ b/_modules/druid_hook.html
@@ -165,6 +165,7 @@
<span class="n">LOAD_CHECK_INTERVAL</span> <span class="o">=</span> <span class="mi">5</span>
<span class="n">DEFAULT_TARGET_PARTITION_SIZE</span> <span class="o">=</span> <span class="mi">5000000</span>
+
<span class="k">class</span> <span class="nc">AirflowDruidLoadException</span><span class="p">(</span><span class="n">AirflowException</span><span class="p">):</span>
<span class="k">pass</span>
@@ -260,10 +261,10 @@
<span class="s2">"mapreduce.map.output.compress"</span><span class="p">:</span> <span class="s2">"false"</span><span class="p">,</span>
<span class="s2">"mapreduce.output.fileoutputformat.compress"</span><span class="p">:</span> <span class="s2">"false"</span><span class="p">,</span>
<span class="p">},</span>
- <span class="s2">"partitionsSpec"</span> <span class="p">:</span> <span class="p">{</span>
- <span class="s2">"type"</span> <span class="p">:</span> <span class="s2">"hashed"</span><span class="p">,</span>
- <span class="s2">"targetPartitionSize"</span> <span class="p">:</span> <span class="n">target_partition_size</span><span class="p">,</span>
- <span class="s2">"numShards"</span> <span class="p">:</span> <span class="n">num_shards</span><span class="p">,</span>
+ <span class="s2">"partitionsSpec"</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s2">"type"</span><span class="p">:</span> <span class="s2">"hashed"</span><span class="p">,</span>
+ <span class="s2">"targetPartitionSize"</span><span class="p">:</span> <span class="n">target_partition_size</span><span class="p">,</span>
+ <span class="s2">"numShards"</span><span class="p">:</span> <span class="n">num_shards</span><span class="p">,</span>
<span class="p">},</span>
<span class="p">},</span>
<span class="s2">"ioConfig"</span><span class="p">:</span> <span class="p">{</span>
@@ -303,8 +304,9 @@
<span class="n">intervals</span><span class="p">,</span> <span class="n">num_shards</span><span class="p">,</span> <span class="n">target_partition_size</span><span class="p">,</span> <span class="n">metric_spec</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">hadoop_dependency_coordinates</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> load data to druid from hdfs</span>
-<span class="sd"> :params ts_dim: The column name to use as a timestamp</span>
-<span class="sd"> :params metric_spec: A list of dictionaries</span>
+
+<span class="sd"> :param ts_dim: The column name to use as a timestamp</span>
+<span class="sd"> :param metric_spec: A list of dictionaries</span>
<span class="sd"> """</span>
<span class="n">task_id</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">send_ingest_query</span><span class="p">(</span>
<span class="n">datasource</span><span class="p">,</span> <span class="n">static_path</span><span class="p">,</span> <span class="n">ts_dim</span><span class="p">,</span> <span class="n">columns</span><span class="p">,</span> <span class="n">metric_spec</span><span class="p">,</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/email_operator.html
----------------------------------------------------------------------
diff --git a/_modules/email_operator.html b/_modules/email_operator.html
index 6bb4434..4d91901 100644
--- a/_modules/email_operator.html
+++ b/_modules/email_operator.html
@@ -169,6 +169,10 @@
<span class="sd"> :type html_content: string</span>
<span class="sd"> :param files: file names to attach in email</span>
<span class="sd"> :type files: list</span>
+<span class="sd"> :param cc: list of recipients to be added in CC field</span>
+<span class="sd"> :type cc: list or string (comma or semicolon delimited)</span>
+<span class="sd"> :param bcc: list of recipients to be added in BCC field</span>
+<span class="sd"> :type bcc: list or string (comma or semicolon delimited)</span>
<span class="sd"> """</span>
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'subject'</span><span class="p">,</span> <span class="s1">'html_content'</span><span class="p">)</span>
@@ -182,15 +186,19 @@
<span class="n">subject</span><span class="p">,</span>
<span class="n">html_content</span><span class="p">,</span>
<span class="n">files</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+ <span class="n">cc</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+ <span class="n">bcc</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">EmailOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">to</span> <span class="o">=</span> <span class="n">to</span>
<span class="bp">self</span><span class="o">.</span><span class="n">subject</span> <span class="o">=</span> <span class="n">subject</span>
<span class="bp">self</span><span class="o">.</span><span class="n">html_content</span> <span class="o">=</span> <span class="n">html_content</span>
<span class="bp">self</span><span class="o">.</span><span class="n">files</span> <span class="o">=</span> <span class="n">files</span> <span class="ow">or</span> <span class="p">[]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cc</span> <span class="o">=</span> <span class="n">cc</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bcc</span> <span class="o">=</span> <span class="n">bcc</span>
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
- <span class="n">send_email</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">subject</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">html_content</span><span class="p">,</span> <span class="n">files</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">files</span><span class="p">)</span></div>
+ <span class="n">send_email</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">subject</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">html_content</span><span class="p">,</span> <span class="n">files</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">files</span><span class="p">,</span> <span class="n">cc</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cc</span><span class="p">,</span> <span class="n">bcc</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bcc</span><span class="p">)</span></div>
</pre></div>
</div>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/ftp_hook.html
----------------------------------------------------------------------
diff --git a/_modules/ftp_hook.html b/_modules/ftp_hook.html
index 19a7498..eed5b97 100644
--- a/_modules/ftp_hook.html
+++ b/_modules/ftp_hook.html
@@ -352,6 +352,16 @@
<span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
<span class="n">conn</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">path</span><span class="p">)</span></div>
+<div class="viewcode-block" id="FTPHook.rename"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.FTPHook.rename">[docs]</a> <span class="k">def</span> <span class="nf">rename</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">from_name</span><span class="p">,</span> <span class="n">to_name</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Rename a file.</span>
+
+<span class="sd"> :param from_name: rename file from name</span>
+<span class="sd"> :param to_name: rename file to name</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">conn</span><span class="o">.</span><span class="n">rename</span><span class="p">(</span><span class="n">from_name</span><span class="p">,</span> <span class="n">to_name</span><span class="p">)</span></div>
+
<span class="k">def</span> <span class="nf">get_mod_time</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
<span class="n">ftp_mdtm</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">sendcmd</span><span class="p">(</span><span class="s1">'MDTM '</span> <span class="o">+</span> <span class="n">path</span><span class="p">)</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/hive_hooks.html
----------------------------------------------------------------------
diff --git a/_modules/hive_hooks.html b/_modules/hive_hooks.html
index e25ec97..926b54c 100644
--- a/_modules/hive_hooks.html
+++ b/_modules/hive_hooks.html
@@ -156,19 +156,25 @@
<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">zip</span>
<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="kn">import</span> <span class="nb">basestring</span>
+<span class="kn">import</span> <span class="nn">collections</span>
<span class="kn">import</span> <span class="nn">unicodecsv</span> <span class="kn">as</span> <span class="nn">csv</span>
+<span class="kn">import</span> <span class="nn">itertools</span>
<span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">import</span> <span class="nn">re</span>
<span class="kn">import</span> <span class="nn">subprocess</span>
+<span class="kn">import</span> <span class="nn">time</span>
<span class="kn">from</span> <span class="nn">tempfile</span> <span class="kn">import</span> <span class="n">NamedTemporaryFile</span>
<span class="kn">import</span> <span class="nn">hive_metastore</span>
<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="kn">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.helpers</span> <span class="kn">import</span> <span class="n">as_flattened_list</span>
<span class="kn">from</span> <span class="nn">airflow.utils.file</span> <span class="kn">import</span> <span class="n">TemporaryDirectory</span>
<span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">configuration</span>
<span class="kn">import</span> <span class="nn">airflow.security.utils</span> <span class="kn">as</span> <span class="nn">utils</span>
+<span class="n">HIVE_QUEUE_PRIORITIES</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'VERY_HIGH'</span><span class="p">,</span> <span class="s1">'HIGH'</span><span class="p">,</span> <span class="s1">'NORMAL'</span><span class="p">,</span> <span class="s1">'LOW'</span><span class="p">,</span> <span class="s1">'VERY_LOW'</span><span class="p">]</span>
+
<div class="viewcode-block" id="HiveCliHook"><a class="viewcode-back" href="../code.html#airflow.hooks.HiveCliHook">[docs]</a><span class="k">class</span> <span class="nc">HiveCliHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
@@ -186,12 +192,24 @@
<span class="sd"> The extra connection parameter ``auth`` gets passed as in the ``jdbc``</span>
<span class="sd"> connection string as is.</span>
+
+<span class="sd"> :param mapred_queue: queue used by the Hadoop Scheduler (Capacity or Fair)</span>
+<span class="sd"> :type mapred_queue: string</span>
+<span class="sd"> :param mapred_queue_priority: priority within the job queue.</span>
+<span class="sd"> Possible settings include: VERY_HIGH, HIGH, NORMAL, LOW, VERY_LOW</span>
+<span class="sd"> :type mapred_queue_priority: string</span>
+<span class="sd"> :param mapred_job_name: This name will appear in the jobtracker.</span>
+<span class="sd"> This can make monitoring easier.</span>
+<span class="sd"> :type mapred_job_name: string</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="s2">"hive_cli_default"</span><span class="p">,</span>
- <span class="n">run_as</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+ <span class="n">run_as</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+ <span class="n">mapred_queue</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+ <span class="n">mapred_queue_priority</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+ <span class="n">mapred_job_name</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">hive_cli_conn_id</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_params</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'hive_cli_params'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">use_beeline</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'use_beeline'</span><span class="p">,</span> <span class="bp">False</span><span class="p">)</span>
@@ -199,16 +217,92 @@
<span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="n">conn</span>
<span class="bp">self</span><span class="o">.</span><span class="n">run_as</span> <span class="o">=</span> <span class="n">run_as</span>
+ <span class="k">if</span> <span class="n">mapred_queue_priority</span><span class="p">:</span>
+ <span class="n">mapred_queue_priority</span> <span class="o">=</span> <span class="n">mapred_queue_priority</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
+ <span class="k">if</span> <span class="n">mapred_queue_priority</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">HIVE_QUEUE_PRIORITIES</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s2">"Invalid Mapred Queue Priority. Valid values are: "</span>
+ <span class="s2">"{}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">HIVE_QUEUE_PRIORITIES</span><span class="p">)))</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue</span> <span class="o">=</span> <span class="n">mapred_queue</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue_priority</span> <span class="o">=</span> <span class="n">mapred_queue_priority</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mapred_job_name</span> <span class="o">=</span> <span class="n">mapred_job_name</span>
+
+ <span class="k">def</span> <span class="nf">_prepare_cli_cmd</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> This function creates the command list from available information</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span>
+ <span class="n">hive_bin</span> <span class="o">=</span> <span class="s1">'hive'</span>
+ <span class="n">cmd_extra</span> <span class="o">=</span> <span class="p">[]</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_beeline</span><span class="p">:</span>
+ <span class="n">hive_bin</span> <span class="o">=</span> <span class="s1">'beeline'</span>
+ <span class="n">jdbc_url</span> <span class="o">=</span> <span class="s2">"jdbc:hive2://{conn.host}:{conn.port}/{conn.schema}"</span>
+ <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'security'</span><span class="p">)</span> <span class="o">==</span> <span class="s1">'kerberos'</span><span class="p">:</span>
+ <span class="n">template</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="s1">'principal'</span><span class="p">,</span> <span class="s2">"hive/_HOST@EXAMPLE.COM"</span><span class="p">)</span>
+ <span class="k">if</span> <span class="s2">"_HOST"</span> <span class="ow">in</span> <span class="n">template</span><span class="p">:</span>
+ <span class="n">template</span> <span class="o">=</span> <span class="n">utils</span><span class="o">.</span><span class="n">replace_hostname_pattern</span><span class="p">(</span>
+ <span class="n">utils</span><span class="o">.</span><span class="n">get_components</span><span class="p">(</span><span class="n">template</span><span class="p">))</span>
+
+ <span class="n">proxy_user</span> <span class="o">=</span> <span class="s2">""</span> <span class="c1"># noqa</span>
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'proxy_user'</span><span class="p">)</span> <span class="o">==</span> <span class="s2">"login"</span> <span class="ow">and</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">:</span>
+ <span class="n">proxy_user</span> <span class="o">=</span> <span class="s2">"hive.server2.proxy.user={0}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'proxy_user'</span><span class="p">)</span> <span class="o">==</span> <span class="s2">"owner"</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_as</span><span class="p">:</span>
+ <span class="n">proxy_user</span> <span class="o">=</span> <span class="s2">"hive.server2.proxy.user={0}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">run_as</span><span class="p">)</span>
+
+ <span class="n">jdbc_url</span> <span class="o">+=</span> <span class="s2">";principal={template};{proxy_user}"</span>
+ <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">auth</span><span class="p">:</span>
+ <span class="n">jdbc_url</span> <span class="o">+=</span> <span class="s2">";auth="</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">auth</span>
+
+ <span class="n">jdbc_url</span> <span class="o">=</span> <span class="n">jdbc_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
+
+ <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'-u'</span><span class="p">,</span> <span class="n">jdbc_url</span><span class="p">]</span>
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">:</span>
+ <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'-n'</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">]</span>
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">:</span>
+ <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'-p'</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">]</span>
+
+ <span class="n">hive_params_list</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_params</span><span class="o">.</span><span class="n">split</span><span class="p">()</span>
+
+ <span class="k">return</span> <span class="p">[</span><span class="n">hive_bin</span><span class="p">]</span> <span class="o">+</span> <span class="n">cmd_extra</span> <span class="o">+</span> <span class="n">hive_params_list</span>
+
+ <span class="k">def</span> <span class="nf">_prepare_hiveconf</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">d</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> This function prepares a list of hiveconf params</span>
+<span class="sd"> from a dictionary of key value pairs.</span>
+
+<span class="sd"> :param d:</span>
+<span class="sd"> :type d: dict</span>
+
+<span class="sd"> >>> hh = HiveCliHook()</span>
+<span class="sd"> >>> hive_conf = {"hive.exec.dynamic.partition": "true",</span>
+<span class="sd"> ... "hive.exec.dynamic.partition.mode": "nonstrict"}</span>
+<span class="sd"> >>> hh._prepare_hiveconf(hive_conf)</span>
+<span class="sd"> ["-hiveconf", "hive.exec.dynamic.partition=true",\</span>
+<span class="sd"> "-hiveconf", "hive.exec.dynamic.partition.mode=nonstrict"]</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">d</span><span class="p">:</span>
+ <span class="k">return</span> <span class="p">[]</span>
+ <span class="k">return</span> <span class="n">as_flattened_list</span><span class="p">(</span>
+ <span class="n">itertools</span><span class="o">.</span><span class="n">izip</span><span class="p">(</span>
+ <span class="p">[</span><span class="s2">"-hiveconf"</span><span class="p">]</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">d</span><span class="p">),</span>
+ <span class="p">[</span><span class="s2">"{}={}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="n">v</span><span class="p">)</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">d</span><span class="o">.</span><span class="n">items</span><span class="p">()]</span>
+ <span class="p">)</span>
+ <span class="p">)</span>
+
<div class="viewcode-block" id="HiveCliHook.run_cli"><a class="viewcode-back" href="../code.html#airflow.hooks.HiveCliHook.run_cli">[docs]</a> <span class="k">def</span> <span class="nf">run_cli</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hql</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span> <span class="n">hive_conf</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="sd">"""</span>
-<span class="sd"> Run an hql statement using the hive cli. If hive_conf is specified it should be a</span>
-<span class="sd"> dict and the entries will be set as key/value pairs in HiveConf</span>
+<span class="sd"> Run an hql statement using the hive cli. If hive_conf is specified</span>
+<span class="sd"> it should be a dict and the entries will be set as key/value pairs</span>
+<span class="sd"> in HiveConf</span>
-<span class="sd"> :param hive_conf: if specified these key value pairs will be passed to hive as</span>
-<span class="sd"> ``-hiveconf "key"="value"``. Note that they will be passed after the</span>
-<span class="sd"> ``hive_cli_params`` and thus will override whatever values are specified in</span>
-<span class="sd"> the database.</span>
+<span class="sd"> :param hive_conf: if specified these key value pairs will be passed</span>
+<span class="sd"> to hive as ``-hiveconf "key"="value"``. Note that they will be</span>
+<span class="sd"> passed after the ``hive_cli_params`` and thus will override</span>
+<span class="sd"> whatever values are specified in the database.</span>
<span class="sd"> :type hive_conf: dict</span>
<span class="sd"> >>> hh = HiveCliHook()</span>
@@ -225,47 +319,29 @@
<span class="k">with</span> <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="nb">dir</span><span class="o">=</span><span class="n">tmp_dir</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
<span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">hql</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'UTF-8'</span><span class="p">))</span>
<span class="n">f</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
- <span class="n">fname</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">name</span>
- <span class="n">hive_bin</span> <span class="o">=</span> <span class="s1">'hive'</span>
- <span class="n">cmd_extra</span> <span class="o">=</span> <span class="p">[]</span>
-
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_beeline</span><span class="p">:</span>
- <span class="n">hive_bin</span> <span class="o">=</span> <span class="s1">'beeline'</span>
- <span class="n">jdbc_url</span> <span class="o">=</span> <span class="s2">"jdbc:hive2://{conn.host}:{conn.port}/{conn.schema}"</span>
- <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'security'</span><span class="p">)</span> <span class="o">==</span> <span class="s1">'kerberos'</span><span class="p">:</span>
- <span class="n">template</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
- <span class="s1">'principal'</span><span class="p">,</span> <span class="s2">"hive/_HOST@EXAMPLE.COM"</span><span class="p">)</span>
- <span class="k">if</span> <span class="s2">"_HOST"</span> <span class="ow">in</span> <span class="n">template</span><span class="p">:</span>
- <span class="n">template</span> <span class="o">=</span> <span class="n">utils</span><span class="o">.</span><span class="n">replace_hostname_pattern</span><span class="p">(</span>
- <span class="n">utils</span><span class="o">.</span><span class="n">get_components</span><span class="p">(</span><span class="n">template</span><span class="p">))</span>
-
- <span class="n">proxy_user</span> <span class="o">=</span> <span class="s2">""</span> <span class="c1"># noqa</span>
- <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'proxy_user'</span><span class="p">)</span> <span class="o">==</span> <span class="s2">"login"</span> <span class="ow">and</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">:</span>
- <span class="n">proxy_user</span> <span class="o">=</span> <span class="s2">"hive.server2.proxy.user={0}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">)</span>
- <span class="k">elif</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'proxy_user'</span><span class="p">)</span> <span class="o">==</span> <span class="s2">"owner"</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_as</span><span class="p">:</span>
- <span class="n">proxy_user</span> <span class="o">=</span> <span class="s2">"hive.server2.proxy.user={0}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">run_as</span><span class="p">)</span>
-
- <span class="n">jdbc_url</span> <span class="o">+=</span> <span class="s2">";principal={template};{proxy_user}"</span>
- <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">auth</span><span class="p">:</span>
- <span class="n">jdbc_url</span> <span class="o">+=</span> <span class="s2">";auth="</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">auth</span>
-
- <span class="n">jdbc_url</span> <span class="o">=</span> <span class="n">jdbc_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
-
- <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'-u'</span><span class="p">,</span> <span class="n">jdbc_url</span><span class="p">]</span>
- <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">:</span>
- <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'-n'</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">]</span>
- <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">:</span>
- <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'-p'</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">]</span>
-
- <span class="n">hive_conf</span> <span class="o">=</span> <span class="n">hive_conf</span> <span class="ow">or</span> <span class="p">{}</span>
- <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">hive_conf</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
- <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'-hiveconf'</span><span class="p">,</span> <span class="s1">'{0}={1}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">)]</span>
-
- <span class="n">hive_cmd</span> <span class="o">=</span> <span class="p">[</span><span class="n">hive_bin</span><span class="p">,</span> <span class="s1">'-f'</span><span class="p">,</span> <span class="n">fname</span><span class="p">]</span> <span class="o">+</span> <span class="n">cmd_extra</span>
-
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_params</span><span class="p">:</span>
- <span class="n">hive_params_list</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_params</span><span class="o">.</span><span class="n">split</span><span class="p">()</span>
- <span class="n">hive_cmd</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">hive_params_list</span><span class="p">)</span>
+ <span class="n">hive_cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_cli_cmd</span><span class="p">()</span>
+ <span class="n">hive_conf_params</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_hiveconf</span><span class="p">(</span><span class="n">hive_conf</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue</span><span class="p">:</span>
+ <span class="n">hive_conf_params</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span>
+ <span class="p">[</span><span class="s1">'-hiveconf'</span><span class="p">,</span>
+ <span class="s1">'mapreduce.job.queuename={}'</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue</span><span class="p">)])</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue_priority</span><span class="p">:</span>
+ <span class="n">hive_conf_params</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span>
+ <span class="p">[</span><span class="s1">'-hiveconf'</span><span class="p">,</span>
+ <span class="s1">'mapreduce.job.priority={}'</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue_priority</span><span class="p">)])</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mapred_job_name</span><span class="p">:</span>
+ <span class="n">hive_conf_params</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span>
+ <span class="p">[</span><span class="s1">'-hiveconf'</span><span class="p">,</span>
+ <span class="s1">'mapred.job.name={}'</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mapred_job_name</span><span class="p">)])</span>
+
+ <span class="n">hive_cmd</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">hive_conf_params</span><span class="p">)</span>
+ <span class="n">hive_cmd</span><span class="o">.</span><span class="n">extend</span><span class="p">([</span><span class="s1">'-f'</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="p">])</span>
+
<span class="k">if</span> <span class="n">verbose</span><span class="p">:</span>
<span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">" "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">hive_cmd</span><span class="p">))</span>
<span class="n">sp</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">Popen</span><span class="p">(</span>
@@ -399,6 +475,8 @@
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">'sp'</span><span class="p">):</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">sp</span><span class="o">.</span><span class="n">poll</span><span class="p">()</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="k">print</span><span class="p">(</span><span class="s2">"Killing the Hive job"</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sp</span><span class="o">.</span><span class="n">terminate</span><span class="p">()</span>
+ <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">60</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">sp</span><span class="o">.</span><span class="n">kill</span><span class="p">()</span></div>
@@ -700,11 +778,12 @@
<span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">hql</span><span class="p">)</span>
<span class="n">schema</span> <span class="o">=</span> <span class="n">cur</span><span class="o">.</span><span class="n">description</span>
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">csv_filepath</span><span class="p">,</span> <span class="s1">'wb'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
- <span class="n">writer</span> <span class="o">=</span> <span class="n">csv</span><span class="o">.</span><span class="n">writer</span><span class="p">(</span><span class="n">f</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">,</span>
- <span class="n">lineterminator</span><span class="o">=</span><span class="n">lineterminator</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">'utf-8'</span><span class="p">)</span>
+ <span class="n">writer</span> <span class="o">=</span> <span class="n">csv</span><span class="o">.</span><span class="n">writer</span><span class="p">(</span><span class="n">f</span><span class="p">,</span>
+ <span class="n">delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">,</span>
+ <span class="n">lineterminator</span><span class="o">=</span><span class="n">lineterminator</span><span class="p">,</span>
+ <span class="n">encoding</span><span class="o">=</span><span class="s1">'utf-8'</span><span class="p">)</span>
<span class="k">if</span> <span class="n">output_header</span><span class="p">:</span>
- <span class="n">writer</span><span class="o">.</span><span class="n">writerow</span><span class="p">([</span><span class="n">c</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
- <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">cur</span><span class="o">.</span><span class="n">description</span><span class="p">])</span>
+ <span class="n">writer</span><span class="o">.</span><span class="n">writerow</span><span class="p">([</span><span class="n">c</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">cur</span><span class="o">.</span><span class="n">description</span><span class="p">])</span>
<span class="n">i</span> <span class="o">=</span> <span class="mi">0</span>
<span class="k">while</span> <span class="bp">True</span><span class="p">:</span>
<span class="n">rows</span> <span class="o">=</span> <span class="p">[</span><span class="n">row</span> <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">cur</span><span class="o">.</span><span class="n">fetchmany</span><span class="p">(</span><span class="n">fetch_size</span><span class="p">)</span> <span class="k">if</span> <span class="n">row</span><span class="p">]</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/hive_operator.html
----------------------------------------------------------------------
diff --git a/_modules/hive_operator.html b/_modules/hive_operator.html
index de5e44b..4d8c62c 100644
--- a/_modules/hive_operator.html
+++ b/_modules/hive_operator.html
@@ -177,6 +177,14 @@
<span class="sd"> :param script_begin_tag: If defined, the operator will get rid of the</span>
<span class="sd"> part of the script before the first occurrence of `script_begin_tag`</span>
<span class="sd"> :type script_begin_tag: str</span>
+<span class="sd"> :param mapred_queue: queue used by the Hadoop CapacityScheduler</span>
+<span class="sd"> :type mapred_queue: string</span>
+<span class="sd"> :param mapred_queue_priority: priority within CapacityScheduler queue.</span>
+<span class="sd"> Possible settings include: VERY_HIGH, HIGH, NORMAL, LOW, VERY_LOW</span>
+<span class="sd"> :type mapred_queue_priority: string</span>
+<span class="sd"> :param mapred_job_name: This name will appear in the jobtracker.</span>
+<span class="sd"> This can make monitoring easier.</span>
+<span class="sd"> :type mapred_job_name: string</span>
<span class="sd"> """</span>
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'hql'</span><span class="p">,</span> <span class="s1">'schema'</span><span class="p">)</span>
@@ -191,6 +199,9 @@
<span class="n">hiveconf_jinja_translate</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
<span class="n">script_begin_tag</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
<span class="n">run_as_owner</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+ <span class="n">mapred_queue</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+ <span class="n">mapred_queue_priority</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+ <span class="n">mapred_job_name</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">HiveOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
@@ -203,8 +214,17 @@
<span class="k">if</span> <span class="n">run_as_owner</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">run_as</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">owner</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue</span> <span class="o">=</span> <span class="n">mapred_queue</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue_priority</span> <span class="o">=</span> <span class="n">mapred_queue_priority</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mapred_job_name</span> <span class="o">=</span> <span class="n">mapred_job_name</span>
+
<span class="k">def</span> <span class="nf">get_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="k">return</span> <span class="n">HiveCliHook</span><span class="p">(</span><span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_conn_id</span><span class="p">,</span> <span class="n">run_as</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">run_as</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">HiveCliHook</span><span class="p">(</span>
+ <span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_conn_id</span><span class="p">,</span>
+ <span class="n">run_as</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">run_as</span><span class="p">,</span>
+ <span class="n">mapred_queue</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue</span><span class="p">,</span>
+ <span class="n">mapred_queue_priority</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue_priority</span><span class="p">,</span>
+ <span class="n">mapred_job_name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mapred_job_name</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">prepare_template</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">hiveconf_jinja_translate</span><span class="p">:</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/hive_to_mysql.html
----------------------------------------------------------------------
diff --git a/_modules/hive_to_mysql.html b/_modules/hive_to_mysql.html
index 867023b..fd8b342 100644
--- a/_modules/hive_to_mysql.html
+++ b/_modules/hive_to_mysql.html
@@ -160,6 +160,7 @@
<span class="kn">from</span> <span class="nn">tempfile</span> <span class="kn">import</span> <span class="n">NamedTemporaryFile</span>
+
<div class="viewcode-block" id="HiveToMySqlTransfer"><a class="viewcode-back" href="../code.html#airflow.operators.HiveToMySqlTransfer">[docs]</a><span class="k">class</span> <span class="nc">HiveToMySqlTransfer</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Moves data from Hive to MySQL, note that for now the data is loaded</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/mysql_hook.html
----------------------------------------------------------------------
diff --git a/_modules/mysql_hook.html b/_modules/mysql_hook.html
index 941737f..a07aeb2 100644
--- a/_modules/mysql_hook.html
+++ b/_modules/mysql_hook.html
@@ -156,6 +156,7 @@
<span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span>
+
<div class="viewcode-block" id="MySqlHook"><a class="viewcode-back" href="../code.html#airflow.hooks.MySqlHook">[docs]</a><span class="k">class</span> <span class="nc">MySqlHook</span><span class="p">(</span><span class="n">DbApiHook</span><span class="p">):</span>
<span class="sd">'''</span>
<span class="sd"> Interact with MySQL.</span>