You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by as...@apache.org on 2019/11/27 19:10:24 UTC
[airflow-site] 11/30: 1.10.0 with Updated Api Reference
This is an automated email from the ASF dual-hosted git repository.
ash pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/airflow-site.git
commit 7d4d76286979a981e36cc5466b2d45d2dd0e4ee2
Author: Kaxil Naik <ka...@gmail.com>
AuthorDate: Wed Aug 29 21:24:05 2018 +0100
1.10.0 with Updated Api Reference
---
.../airflow/contrib/executors/mesos_executor.html | 104 +-
.../airflow/contrib/hooks/aws_dynamodb_hook.html | 55 +-
_modules/airflow/contrib/hooks/aws_hook.html | 61 +-
.../airflow/contrib/hooks/aws_lambda_hook.html | 52 +-
.../contrib/hooks/azure_data_lake_hook.html | 359 ++
.../contrib/hooks/azure_fileshare_hook.html | 430 ++
_modules/airflow/contrib/hooks/bigquery_hook.html | 166 +-
_modules/airflow/contrib/hooks/cassandra_hook.html | 400 ++
_modules/airflow/contrib/hooks/cloudant_hook.html | 301 ++
.../airflow/contrib/hooks/databricks_hook.html | 60 +-
_modules/airflow/contrib/hooks/datadog_hook.html | 63 +-
_modules/airflow/contrib/hooks/datastore_hook.html | 88 +-
.../contrib/hooks/discord_webhook_hook.html | 49 +-
_modules/airflow/contrib/hooks/emr_hook.html | 55 +-
_modules/airflow/contrib/hooks/fs_hook.html | 53 +-
_modules/airflow/contrib/hooks/ftp_hook.html | 51 +-
.../airflow/contrib/hooks/gcp_api_base_hook.html | 101 +-
.../airflow/contrib/hooks/gcp_container_hook.html | 435 ++
.../airflow/contrib/hooks/gcp_dataflow_hook.html | 52 +-
.../airflow/contrib/hooks/gcp_dataproc_hook.html | 65 +-
.../airflow/contrib/hooks/gcp_mlengine_hook.html | 50 +-
.../airflow/contrib/hooks/gcp_pubsub_hook.html | 52 +-
_modules/airflow/contrib/hooks/gcs_hook.html | 74 +-
_modules/airflow/contrib/hooks/jenkins_hook.html | 49 +-
_modules/airflow/contrib/hooks/jira_hook.html | 56 +-
_modules/airflow/contrib/hooks/mongo_hook.html | 337 ++
_modules/airflow/contrib/hooks/pinot_hook.html | 57 +-
_modules/airflow/contrib/hooks/qubole_hook.html | 59 +-
_modules/airflow/contrib/hooks/redis_hook.html | 49 +-
_modules/airflow/contrib/hooks/redshift_hook.html | 49 +-
_modules/airflow/contrib/hooks/segment_hook.html | 310 ++
_modules/airflow/contrib/hooks/sftp_hook.html | 49 +-
.../airflow/contrib/hooks/slack_webhook_hook.html | 49 +-
_modules/airflow/contrib/hooks/snowflake_hook.html | 316 ++
.../airflow/contrib/hooks/spark_jdbc_hook.html | 49 +-
_modules/airflow/contrib/hooks/spark_sql_hook.html | 56 +-
.../airflow/contrib/hooks/spark_submit_hook.html | 52 +-
_modules/airflow/contrib/hooks/sqoop_hook.html | 70 +-
_modules/airflow/contrib/hooks/ssh_hook.html | 55 +-
_modules/airflow/contrib/hooks/vertica_hook.html | 53 +-
_modules/airflow/contrib/hooks/wasb_hook.html | 49 +-
_modules/airflow/contrib/hooks/winrm_hook.html | 355 ++
_modules/airflow/contrib/kubernetes/secret.html | 45 +-
.../contrib/operators/awsbatch_operator.html | 88 +-
.../contrib/operators/bigquery_check_operator.html | 52 +-
.../contrib/operators/bigquery_get_data.html | 55 +-
.../contrib/operators/bigquery_operator.html | 121 +-
.../operators/bigquery_table_delete_operator.html | 51 +-
.../contrib/operators/bigquery_to_bigquery.html | 56 +-
.../airflow/contrib/operators/bigquery_to_gcs.html | 55 +-
.../contrib/operators/cassandra_to_gcs.html | 573 ++
.../contrib/operators/databricks_operator.html | 63 +-
.../contrib/operators/dataflow_operator.html | 49 +-
.../contrib/operators/dataproc_operator.html | 306 +-
.../operators/datastore_export_operator.html | 60 +-
.../operators/datastore_import_operator.html | 54 +-
.../operators/discord_webhook_operator.html | 53 +-
.../{vertica_operator.html => druid_operator.html} | 116 +-
.../airflow/contrib/operators/ecs_operator.html | 78 +-
.../contrib/operators/emr_add_steps_operator.html | 53 +-
.../operators/emr_create_job_flow_operator.html | 55 +-
.../operators/emr_terminate_job_flow_operator.html | 51 +-
.../airflow/contrib/operators/file_to_gcs.html | 59 +-
.../airflow/contrib/operators/file_to_wasb.html | 61 +-
.../contrib/operators/gcp_container_operator.html | 390 ++
.../contrib/operators/gcs_download_operator.html | 64 +-
.../contrib/operators/gcs_list_operator.html | 56 +-
.../airflow/contrib/operators/gcs_operator.html | 57 +-
_modules/airflow/contrib/operators/gcs_to_bq.html | 64 +-
_modules/airflow/contrib/operators/gcs_to_gcs.html | 150 +-
_modules/airflow/contrib/operators/gcs_to_s3.html | 58 +-
.../contrib/operators/hipchat_operator.html | 57 +-
.../contrib/operators/hive_to_dynamodb.html | 327 ++
.../operators/jenkins_job_trigger_operator.html | 51 +-
.../airflow/contrib/operators/jira_operator.html | 51 +-
.../contrib/operators/kubernetes_pod_operator.html | 111 +-
.../contrib/operators/mlengine_operator.html | 94 +-
.../airflow/contrib/operators/mongo_to_s3.html | 337 ++
.../airflow/contrib/operators/mysql_to_gcs.html | 48 +-
.../operators/postgres_to_gcs_operator.html | 49 +-
.../airflow/contrib/operators/pubsub_operator.html | 52 +-
.../contrib/operators/qubole_check_operator.html | 443 ++
.../airflow/contrib/operators/qubole_operator.html | 51 +-
.../contrib/operators/s3_list_operator.html | 55 +-
.../contrib/operators/s3_to_gcs_operator.html | 53 +-
...ator.html => segment_track_event_operator.html} | 126 +-
.../airflow/contrib/operators/sftp_operator.html | 56 +-
.../contrib/operators/slack_webhook_operator.html | 49 +-
...rtica_operator.html => snowflake_operator.html} | 97 +-
.../contrib/operators/spark_jdbc_operator.html | 49 +-
.../contrib/operators/spark_sql_operator.html | 57 +-
.../contrib/operators/spark_submit_operator.html | 52 +-
.../airflow/contrib/operators/sqoop_operator.html | 69 +-
.../airflow/contrib/operators/ssh_operator.html | 55 +-
.../contrib/operators/vertica_operator.html | 51 +-
.../airflow/contrib/operators/vertica_to_hive.html | 357 ++
.../airflow/contrib/operators/winrm_operator.html | 328 ++
.../sensors/aws_redshift_cluster_sensor.html | 53 +-
_modules/airflow/contrib/sensors/bash_sensor.html | 50 +-
.../airflow/contrib/sensors/bigquery_sensor.html | 94 +-
.../airflow/contrib/sensors/datadog_sensor.html | 53 +-
.../airflow/contrib/sensors/emr_base_sensor.html | 53 +-
.../contrib/sensors/emr_job_flow_sensor.html | 52 +-
.../airflow/contrib/sensors/emr_step_sensor.html | 49 +-
_modules/airflow/contrib/sensors/file_sensor.html | 53 +-
_modules/airflow/contrib/sensors/ftp_sensor.html | 53 +-
_modules/airflow/contrib/sensors/gcs_sensor.html | 160 +-
_modules/airflow/contrib/sensors/hdfs_sensor.html | 49 +-
_modules/airflow/contrib/sensors/jira_sensor.html | 65 +-
.../airflow/contrib/sensors/pubsub_sensor.html | 53 +-
.../airflow/contrib/sensors/qubole_sensor.html | 61 +-
.../airflow/contrib/sensors/redis_key_sensor.html | 53 +-
_modules/airflow/contrib/sensors/sftp_sensor.html | 55 +-
_modules/airflow/contrib/sensors/wasb_sensor.html | 57 +-
_modules/airflow/executors/celery_executor.html | 81 +-
_modules/airflow/executors/local_executor.html | 65 +-
.../airflow/executors/sequential_executor.html | 61 +-
_modules/airflow/hooks/S3_hook.html | 98 +-
_modules/airflow/hooks/dbapi_hook.html | 80 +-
_modules/airflow/hooks/docker_hook.html | 303 ++
_modules/airflow/hooks/druid_hook.html | 377 ++
_modules/airflow/hooks/hdfs_hook.html | 64 +-
_modules/airflow/hooks/hive_hooks.html | 1098 ++++
_modules/airflow/hooks/http_hook.html | 175 +-
.../hooks/{mssql_hook.html => jdbc_hook.html} | 106 +-
_modules/airflow/hooks/mssql_hook.html | 53 +-
_modules/airflow/hooks/mysql_hook.html | 83 +-
_modules/airflow/hooks/oracle_hook.html | 91 +-
_modules/airflow/hooks/pig_hook.html | 49 +-
_modules/airflow/hooks/postgres_hook.html | 94 +-
_modules/airflow/hooks/presto_hook.html | 56 +-
.../hooks/{mssql_hook.html => samba_hook.html} | 109 +-
_modules/airflow/hooks/slack_hook.html | 52 +-
_modules/airflow/hooks/sqlite_hook.html | 49 +-
_modules/airflow/hooks/webhdfs_hook.html | 63 +-
_modules/airflow/hooks/zendesk_hook.html | 329 ++
_modules/airflow/macros.html | 58 +-
_modules/airflow/macros/hive.html | 57 +-
_modules/airflow/models.html | 658 ++-
_modules/airflow/operators/bash_operator.html | 69 +-
_modules/airflow/operators/check_operator.html | 59 +-
_modules/airflow/operators/dagrun_operator.html | 77 +-
_modules/airflow/operators/docker_operator.html | 470 ++
.../airflow/operators/druid_check_operator.html | 309 ++
_modules/airflow/operators/dummy_operator.html | 49 +-
_modules/airflow/operators/email_operator.html | 55 +-
_modules/airflow/operators/generic_transfer.html | 56 +-
_modules/airflow/operators/hive_operator.html | 360 ++
.../airflow/operators/hive_stats_operator.html | 408 ++
_modules/airflow/operators/hive_to_druid.html | 456 ++
.../{presto_to_mysql.html => hive_to_mysql.html} | 126 +-
...s_operator.html => hive_to_samba_operator.html} | 120 +-
_modules/airflow/operators/http_operator.html | 53 +-
.../{postgres_operator.html => jdbc_operator.html} | 91 +-
.../airflow/operators/latest_only_operator.html | 50 +-
_modules/airflow/operators/mssql_operator.html | 53 +-
_modules/airflow/operators/mssql_to_hive.html | 357 ++
_modules/airflow/operators/mysql_operator.html | 52 +-
_modules/airflow/operators/mysql_to_hive.html | 360 ++
_modules/airflow/operators/oracle_operator.html | 52 +-
_modules/airflow/operators/pig_operator.html | 52 +-
_modules/airflow/operators/postgres_operator.html | 51 +-
.../airflow/operators/presto_check_operator.html | 52 +-
_modules/airflow/operators/presto_to_mysql.html | 60 +-
_modules/airflow/operators/python_operator.html | 124 +-
.../airflow/operators/redshift_to_s3_operator.html | 50 +-
.../operators/s3_file_transform_operator.html | 53 +-
.../airflow/operators/s3_to_hive_operator.html | 502 ++
.../airflow/operators/s3_to_redshift_operator.html | 50 +-
_modules/airflow/operators/slack_operator.html | 78 +-
_modules/airflow/operators/sqlite_operator.html | 52 +-
_modules/airflow/operators/subdag_operator.html | 45 +-
_modules/airflow/sensors/base_sensor_operator.html | 67 +-
_modules/airflow/sensors/external_task_sensor.html | 53 +-
_modules/airflow/sensors/hdfs_sensor.html | 53 +-
.../airflow/sensors/hive_partition_sensor.html | 53 +-
_modules/airflow/sensors/http_sensor.html | 53 +-
.../sensors/metastore_partition_sensor.html | 53 +-
.../sensors/named_hive_partition_sensor.html | 49 +-
_modules/airflow/sensors/s3_key_sensor.html | 53 +-
_modules/airflow/sensors/s3_prefix_sensor.html | 53 +-
_modules/airflow/sensors/sql_sensor.html | 53 +-
_modules/airflow/sensors/time_delta_sensor.html | 53 +-
_modules/airflow/sensors/time_sensor.html | 53 +-
_modules/airflow/sensors/web_hdfs_sensor.html | 53 +-
_modules/index.html | 80 +-
_sources/code.rst.txt | 9 +-
_sources/concepts.rst.txt | 20 +-
_sources/howto/index.rst.txt | 1 +
_sources/howto/operator.rst.txt | 103 +
_sources/howto/write-logs.rst.txt | 4 +-
_sources/installation.rst.txt | 67 +-
_sources/integration.rst.txt | 20 +-
_sources/plugins.rst.txt | 20 +
_sources/project.rst.txt | 1 +
_sources/scheduler.rst.txt | 5 +-
_sources/security.rst.txt | 2 +-
_static/basic.css | 24 +-
_static/css/badge_only.css | 3 +-
_static/css/theme.css | 9 +-
_static/doctools.js | 4 +-
_static/documentation_options.js | 9 +
_static/fonts/Inconsolata-Bold.ttf | Bin 109948 -> 0 bytes
_static/fonts/Inconsolata-Regular.ttf | Bin 96964 -> 0 bytes
_static/fonts/Lato-Bold.ttf | Bin 656544 -> 0 bytes
_static/fonts/Lato-Regular.ttf | Bin 656568 -> 0 bytes
_static/fonts/Lato/lato-bold.eot | Bin 0 -> 256056 bytes
_static/fonts/Lato/lato-bold.ttf | Bin 0 -> 600856 bytes
_static/fonts/Lato/lato-bold.woff | Bin 0 -> 309728 bytes
_static/fonts/Lato/lato-bold.woff2 | Bin 0 -> 184912 bytes
_static/fonts/Lato/lato-bolditalic.eot | Bin 0 -> 266158 bytes
_static/fonts/Lato/lato-bolditalic.ttf | Bin 0 -> 622572 bytes
_static/fonts/Lato/lato-bolditalic.woff | Bin 0 -> 323344 bytes
_static/fonts/Lato/lato-bolditalic.woff2 | Bin 0 -> 193308 bytes
_static/fonts/Lato/lato-italic.eot | Bin 0 -> 268604 bytes
_static/fonts/Lato/lato-italic.ttf | Bin 0 -> 639388 bytes
_static/fonts/Lato/lato-italic.woff | Bin 0 -> 328412 bytes
_static/fonts/Lato/lato-italic.woff2 | Bin 0 -> 195704 bytes
_static/fonts/Lato/lato-regular.eot | Bin 0 -> 253461 bytes
_static/fonts/Lato/lato-regular.ttf | Bin 0 -> 607720 bytes
_static/fonts/Lato/lato-regular.woff | Bin 0 -> 309192 bytes
_static/fonts/Lato/lato-regular.woff2 | Bin 0 -> 182708 bytes
_static/fonts/RobotoSlab/roboto-slab-v7-bold.eot | Bin 0 -> 79520 bytes
.../roboto-slab-v7-bold.ttf} | Bin
_static/fonts/RobotoSlab/roboto-slab-v7-bold.woff | Bin 0 -> 87624 bytes
_static/fonts/RobotoSlab/roboto-slab-v7-bold.woff2 | Bin 0 -> 67312 bytes
.../fonts/RobotoSlab/roboto-slab-v7-regular.eot | Bin 0 -> 78331 bytes
.../roboto-slab-v7-regular.ttf} | Bin
.../fonts/RobotoSlab/roboto-slab-v7-regular.woff | Bin 0 -> 86288 bytes
.../fonts/RobotoSlab/roboto-slab-v7-regular.woff2 | Bin 0 -> 66444 bytes
_static/fonts/fontawesome-webfont.eot | Bin 76518 -> 165742 bytes
_static/fonts/fontawesome-webfont.svg | 3350 +++++++++---
_static/fonts/fontawesome-webfont.ttf | Bin 152796 -> 165548 bytes
_static/fonts/fontawesome-webfont.woff | Bin 90412 -> 98024 bytes
_static/fonts/fontawesome-webfont.woff2 | Bin 0 -> 77160 bytes
_static/{jquery-3.1.0.js => jquery-3.2.1.js} | 769 ++-
_static/jquery.js | 8 +-
_static/js/theme.js | 172 +-
_static/websupport.js | 2 +-
api.html | 56 +-
cli.html | 129 +-
code.html | 5620 +++++++++++++++-----
concepts.html | 332 +-
faq.html | 176 +-
genindex.html | 537 +-
howto/executor/use-celery.html | 68 +-
howto/executor/use-dask.html | 58 +-
howto/executor/use-mesos.html | 68 +-
howto/index.html | 53 +-
howto/initialize-database.html | 58 +-
howto/manage-connections.html | 96 +-
howto/operator.html | 400 ++
howto/run-with-systemd.html | 60 +-
howto/run-with-upstart.html | 66 +-
howto/secure-connections.html | 68 +-
howto/set-config.html | 64 +-
howto/use-test-config.html | 54 +-
howto/write-logs.html | 96 +-
index.html | 65 +-
installation.html | 211 +-
integration.html | 1479 ++++--
kubernetes.html | 65 +-
license.html | 48 +-
lineage.html | 50 +-
objects.inv | Bin 6013 -> 7505 bytes
plugins.html | 70 +-
profiling.html | 46 +-
project.html | 51 +-
py-modindex.html | 49 +-
scheduler.html | 132 +-
search.html | 42 +-
searchindex.js | 2 +-
security.html | 112 +-
start.html | 62 +-
timezone.html | 56 +-
tutorial.html | 142 +-
ui.html | 46 +-
277 files changed, 26848 insertions(+), 10956 deletions(-)
diff --git a/_modules/airflow/contrib/executors/mesos_executor.html b/_modules/airflow/contrib/executors/mesos_executor.html
index 312fffe..a495421 100644
--- a/_modules/airflow/contrib/executors/mesos_executor.html
+++ b/_modules/airflow/contrib/executors/mesos_executor.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -193,7 +183,7 @@
<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
<span class="kn">from</span> <span class="nn">airflow.www.utils</span> <span class="k">import</span> <span class="n">LoginMixin</span>
-<span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
+
<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
<span class="kn">from</span> <span class="nn">queue</span> <span class="k">import</span> <span class="n">Queue</span>
@@ -207,7 +197,7 @@
<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span>
<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
-
+<span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
<span class="n">DEFAULT_FRAMEWORK_NAME</span> <span class="o">=</span> <span class="s1">'Airflow'</span>
<span class="n">FRAMEWORK_CONNID_PREFIX</span> <span class="o">=</span> <span class="s1">'mesos_framework_'</span>
@@ -245,7 +235,8 @@
<span class="p">)</span>
<span class="k">def</span> <span class="nf">registered</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">frameworkId</span><span class="p">,</span> <span class="n">masterInfo</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler registered to Mesos with framework ID </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler registered to Mesos with framework ID </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
+ <span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
<span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">)</span> <span class="ow">and</span> \
<span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span>
@@ -299,14 +290,15 @@
<span class="k">elif</span> <span class="n">resource</span><span class="o">.</span><span class="n">name</span> <span class="o">==</span> <span class="s2">"mem"</span><span class="p">:</span>
<span class="n">offerMem</span> <span class="o">+=</span> <span class="n">resource</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Received offer </span><span class="si">%s</span><span class="s2"> with cpus: </span><span class="si">%s</span><span class="s2"> and mem: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">offer</span><span class="o">.</span><span class="n">id</span><s [...]
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Received offer </span><span class="si">%s</span><span class="s2"> with cpus: </span><span class="si">%s</span><span class="s2"> and mem: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
+ <span class="n">offer</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">value</span><span class="p">,</span> <span class="n">offerCpus</span><span class="p">,</span> <span class="n">offerMem</span><span class="p">)</span>
<span class="n">remainingCpus</span> <span class="o">=</span> <span class="n">offerCpus</span>
<span class="n">remainingMem</span> <span class="o">=</span> <span class="n">offerMem</span>
<span class="k">while</span> <span class="p">(</span><span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">empty</span><span class="p">())</span> <span class="ow">and</span> \
- <span class="n">remainingCpus</span> <span class="o">>=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span> <span class="ow">and</span> \
- <span class="n">remainingMem</span> <span class="o">>=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span><span class="p">:</span>
+ <span class="n">remainingCpus</span> <span class="o">>=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span> <span class="ow">and</span> \
+ <span class="n">remainingMem</span> <span class="o">>=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span><span class="p">:</span>
<span class="n">key</span><span class="p">,</span> <span class="n">cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
<span class="n">tid</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_counter</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_counter</span> <span class="o">+=</span> <span class="mi">1</span>
@@ -365,7 +357,8 @@
<span class="k">try</span><span class="p">:</span>
<span class="n">key</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span><span class="p">[</span><span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">]</span>
<span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
- <span class="c1"># The map may not contain an item if the framework re-registered after a failover.</span>
+ <span class="c1"># The map may not contain an item if the framework re-registered</span>
+ <span class="c1"># after a failover.</span>
<span class="c1"># Discard these tasks.</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Unrecognised task key </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
<span class="k">return</span>
@@ -392,7 +385,7 @@
<span class="sd"> elastic distributed systems to easily be built and run effectively.</span>
<span class="sd"> See http://mesos.apache.org/</span>
<span class="sd"> """</span>
- <span class="k">def</span> <span class="nf">start</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<div class="viewcode-block" id="MesosExecutor.start"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor.start">[docs]</a> <span class="k">def</span> <span class="nf">start</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span> <span class="o">=</span> <span class="n">Queue</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="n">Queue</span><span class="p">()</span>
<span class="n">framework</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">FrameworkInfo</span><span class="p">()</span>
@@ -428,7 +421,8 @@
<span class="n">session</span> <span class="o">=</span> <span class="n">Session</span><span class="p">()</span>
<span class="n">connection</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">Connection</span><span class="p">)</span><span class="o">.</span><span class="n">filter_by</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p [...]
<span class="k">if</span> <span class="n">connection</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
- <span class="c1"># Set the Framework ID to let the scheduler reconnect with running tasks.</span>
+ <span class="c1"># Set the Framework ID to let the scheduler reconnect</span>
+ <span class="c1"># with running tasks.</span>
<span class="n">framework</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">connection</span><span class="o">.</span><span class="n">extra</span>
<span class="n">framework</span><span class="o">.</span><span class="n">failover_timeout</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span>
@@ -439,7 +433,8 @@
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
<span class="s1">'MesosFramework master : </span><span class="si">%s</span><span class="s1">, name : </span><span class="si">%s</span><span class="s1">, cpu : </span><span class="si">%s</span><span class="s1">, mem : </span><span class="si">%s</span><span class="s1">, checkpoint : </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
- <span class="n">master</span><span class="p">,</span> <span class="n">framework</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">task_cpu</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">task_memory</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">framework</span><span class="o"> [...]
+ <span class="n">master</span><span class="p">,</span> <span class="n">framework</span><span class="o">.</span><span class="n">name</span><span class="p">,</span>
+ <span class="nb">str</span><span class="p">(</span><span class="n">task_cpu</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">task_memory</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span><span class="p">)</span>
<span class="p">)</span>
<span class="n">implicit_acknowledgements</span> <span class="o">=</span> <span class="mi">1</span>
@@ -447,10 +442,12 @@
<span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'AUTHENTICATE'</span><span class="p">):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_PRINCIPAL'</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Expecting authentication principal in the environment"</span><span class="p">)</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"mesos.default_principal not provided in authenticated mode"</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s2">"mesos.default_principal not provided in authenticated mode"</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_SECRET'</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Expecting authentication secret in the environment"</span><span class="p">)</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"mesos.default_secret not provided in authenticated mode"</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s2">"mesos.default_secret not provided in authenticated mode"</span><span class="p">)</span>
<span class="n">credential</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">Credential</span><span class="p">()</span>
<span class="n">credential</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_PRINCIPAL'</span><span class="p">)</span>
@@ -459,7 +456,10 @@
<span class="n">framework</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="n">credential</span><span class="o">.</span><span class="n">principal</span>
<span class="n">driver</span> <span class="o">=</span> <span class="n">mesos</span><span class="o">.</span><span class="n">native</span><span class="o">.</span><span class="n">MesosSchedulerDriver</span><span class="p">(</span>
- <span class="n">AirflowMesosScheduler</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="p">,</span> <span class="n">task_cpu</span><span class="p">,</span> <span class="n">task_memory</span><span class="p">),</span>
+ <span class="n">AirflowMesosScheduler</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="p">,</span>
+ <span class="n">task_cpu</span><span class="p">,</span>
+ <span class="n">task_memory</span><span class="p">),</span>
<span class="n">framework</span><span class="p">,</span>
<span class="n">master</span><span class="p">,</span>
<span class="n">implicit_acknowledgements</span><span class="p">,</span>
@@ -467,31 +467,32 @@
<span class="k">else</span><span class="p">:</span>
<span class="n">framework</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="s1">'Airflow'</span>
<span class="n">driver</span> <span class="o">=</span> <span class="n">mesos</span><span class="o">.</span><span class="n">native</span><span class="o">.</span><span class="n">MesosSchedulerDriver</span><span class="p">(</span>
- <span class="n">AirflowMesosScheduler</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="p">,</span> <span class="n">task_cpu</span><span class="p">,</span> <span class="n">task_memory</span><span class="p">),</span>
+ <span class="n">AirflowMesosScheduler</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="p">,</span>
+ <span class="n">task_cpu</span><span class="p">,</span>
+ <span class="n">task_memory</span><span class="p">),</span>
<span class="n">framework</span><span class="p">,</span>
<span class="n">master</span><span class="p">,</span>
<span class="n">implicit_acknowledgements</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span> <span class="o">=</span> <span class="n">driver</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span><span class="o">.</span><span class="n">start</span><span class="p">()</span></div>
- <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">executor_config</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">))</span>
+<div class="viewcode-block" id="MesosExecutor.execute_async"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor.execute_async">[docs]</a> <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</ [...]
+ <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">))</span></div>
- <span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<div class="viewcode-block" id="MesosExecutor.sync"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor.sync">[docs]</a> <span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">empty</span><span class="p">():</span>
<span class="n">results</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">change_state</span><span class="p">(</span><span class="o">*</span><span class="n">results</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">change_state</span><span class="p">(</span><span class="o">*</span><span class="n">results</span><span class="p">)</span></div>
- <span class="k">def</span> <span class="nf">end</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<div class="viewcode-block" id="MesosExecutor.end"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor.end">[docs]</a> <span class="k">def</span> <span class="nf">end</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span><span class="o">.</span><span class="n">stop</span><span class="p">()</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span><span class="o">.</span><span class="n">stop</span><span class="p">()</span></div></div>
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -503,7 +504,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -522,6 +523,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -534,19 +536,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html b/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html
index 68c9921..b8fea44 100644
--- a/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html
+++ b/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -204,7 +194,11 @@
<span class="sd"> :type region_name: str</span>
<span class="sd"> """</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table_keys</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">table_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</sp [...]
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">table_keys</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">table_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">table_keys</span> <span class="o">=</span> <span class="n">table_keys</span>
<span class="bp">self</span><span class="o">.</span><span class="n">table_name</span> <span class="o">=</span> <span class="n">table_name</span>
<span class="bp">self</span><span class="o">.</span><span class="n">region_name</span> <span class="o">=</span> <span class="n">region_name</span>
@@ -237,9 +231,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -251,7 +243,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -270,6 +262,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -282,19 +275,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/aws_hook.html b/_modules/airflow/contrib/hooks/aws_hook.html
index 7d9a60f..f572e90 100644
--- a/_modules/airflow/contrib/hooks/aws_hook.html
+++ b/_modules/airflow/contrib/hooks/aws_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -272,13 +262,16 @@
<span class="n">aws_secret_access_key</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">password</span>
<span class="k">elif</span> <span class="s1">'aws_secret_access_key'</span> <span class="ow">in</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">:</span>
- <span class="n">aws_access_key_id</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s1">'aws_access_key_id'</span><span class="p">]</span>
- <span class="n">aws_secret_access_key</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s1">'aws_secret_access_key'</span><span class="p">]</span>
+ <span class="n">aws_access_key_id</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span>
+ <span class="s1">'aws_access_key_id'</span><span class="p">]</span>
+ <span class="n">aws_secret_access_key</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span>
+ <span class="s1">'aws_secret_access_key'</span><span class="p">]</span>
<span class="k">elif</span> <span class="s1">'s3_config_file'</span> <span class="ow">in</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">:</span>
<span class="n">aws_access_key_id</span><span class="p">,</span> <span class="n">aws_secret_access_key</span> <span class="o">=</span> \
- <span class="n">_parse_s3_config</span><span class="p">(</span><span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s1">'s3_config_file'</span><span class="p">],</span>
- <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'s3_config_format'</span><span class="p">))</span>
+ <span class="n">_parse_s3_config</span><span class="p">(</span>
+ <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s1">'s3_config_file'</span><span class="p">],</span>
+ <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'s3_config_format'</span><span class="p">))</span>
<span class="k">if</span> <span class="n">region_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">region_name</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'region_name'</span><span class="p">)</span>
@@ -289,7 +282,6 @@
<span class="k">if</span> <span class="n">role_arn</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">aws_account_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> \
<span class="n">aws_iam_role</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-
<span class="n">role_arn</span> <span class="o">=</span> <span class="s2">"arn:aws:iam::"</span> <span class="o">+</span> <span class="n">aws_account_id</span> <span class="o">+</span> <span class="s2">":role/"</span> <span class="o">+</span> <span class="n">aws_iam_role</span>
<span class="k">if</span> <span class="n">role_arn</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
@@ -347,9 +339,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -361,7 +351,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -380,6 +370,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -392,19 +383,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/aws_lambda_hook.html b/_modules/airflow/contrib/hooks/aws_lambda_hook.html
index 5953a6b..131de2a 100644
--- a/_modules/airflow/contrib/hooks/aws_lambda_hook.html
+++ b/_modules/airflow/contrib/hooks/aws_lambda_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -207,7 +197,8 @@
<span class="sd"> :type invocation_type: str</span>
<span class="sd"> """</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">function_name</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">log_type</span><span class="o">=</span><span class="s1">'None'</span><span class="p">,</span> <span class="n">qualifier</span><span class="o">=</span><span [...]
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">function_name</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">log_type</span><span class="o">=</span><span class="s1">'None'</span><span class="p">,</span> <span class="n">qualifier</span><span class="o">=</span><span class="s1">'$LATEST'</span><span class="p">,</span>
<span class="n">invocation_type</span><span class="o">=</span><span class="s1">'RequestResponse'</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">function_name</span> <span class="o">=</span> <span class="n">function_name</span>
<span class="bp">self</span><span class="o">.</span><span class="n">region_name</span> <span class="o">=</span> <span class="n">region_name</span>
@@ -239,9 +230,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -253,7 +242,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -272,6 +261,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -284,19 +274,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/azure_data_lake_hook.html b/_modules/airflow/contrib/hooks/azure_data_lake_hook.html
new file mode 100644
index 0000000..89e24a5
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/azure_data_lake_hook.html
@@ -0,0 +1,359 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.azure_data_lake_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.azure_data_lake_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.azure_data_lake_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">azure.datalake.store</span> <span class="k">import</span> <span class="n">core</span><span class="p">,</span> <span class="n">lib</span><span class="p">,</span> <span class="n">multithread</span>
+
+
+<div class="viewcode-block" id="AzureDataLakeHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_data_lake_hook.AzureDataLakeHook">[docs]</a><span class="k">class</span> <span class="nc">AzureDataLakeHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interacts with Azure Data Lake.</span>
+
+<span class="sd"> Client ID and client secret should be in user and password parameters.</span>
+<span class="sd"> Tenant and account name should be extra field as</span>
+<span class="sd"> {"tenant": "<TENANT>", "account_name": "ACCOUNT_NAME"}.</span>
+
+<span class="sd"> :param azure_data_lake_conn_id: Reference to the Azure Data Lake connection.</span>
+<span class="sd"> :type azure_data_lake_conn_id: str</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">azure_data_lake_conn_id</span><span class="o">=</span><span class="s1">'azure_data_lake_default'</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span> <span class="o">=</span> <span class="n">azure_data_lake_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">connection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+<div class="viewcode-block" id="AzureDataLakeHook.get_conn"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_data_lake_hook.AzureDataLakeHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""Return a AzureDLFileSystem object."""</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+ <span class="n">service_options</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">account_name</span> <span class="o">=</span> <span class="n">service_options</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'account_name'</span><span class="p">)</span>
+
+ <span class="n">adlCreds</span> <span class="o">=</span> <span class="n">lib</span><span class="o">.</span><span class="n">auth</span><span class="p">(</span><span class="n">tenant_id</span><span class="o">=</span><span class="n">service_options</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'tenant'</span><span class="p">),</span>
+ <span class="n">client_secret</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span>
+ <span class="n">client_id</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">)</span>
+ <span class="n">adlsFileSystemClient</span> <span class="o">=</span> <span class="n">core</span><span class="o">.</span><span class="n">AzureDLFileSystem</span><span class="p">(</span><span class="n">adlCreds</span><span class="p">,</span>
+ <span class="n">store_name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">account_name</span><span class="p">)</span>
+ <span class="n">adlsFileSystemClient</span><span class="o">.</span><span class="n">connect</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">adlsFileSystemClient</span></div>
+
+<div class="viewcode-block" id="AzureDataLakeHook.check_for_file"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_data_lake_hook.AzureDataLakeHook.check_for_file">[docs]</a> <span class="k">def</span> <span class="nf">check_for_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">file_path</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Check if a file exists on Azure Data Lake.</span>
+
+<span class="sd"> :param file_path: Path and name of the file.</span>
+<span class="sd"> :type file_path: str</span>
+<span class="sd"> :return: True if the file exists, False otherwise.</span>
+<span class="sd"> :rtype bool</span>
+<span class="sd"> """</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">files</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">glob</span><span class="p">(</span><span class="n">file_path</span><span class="p">,</span> <span class="n">details</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">invalidate_cache</span><span class="o">=</span><span class="kc">True</span><span clas [...]
+ <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="n">files</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span>
+ <span class="k">except</span> <span class="ne">FileNotFoundError</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">False</span></div>
+
+<div class="viewcode-block" id="AzureDataLakeHook.upload_file"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_data_lake_hook.AzureDataLakeHook.upload_file">[docs]</a> <span class="k">def</span> <span class="nf">upload_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">local_path</span><span class="p">,</span> <span class="n">remote_path</span><span class="p">,</span> <span class="n">nthreads< [...]
+ <span class="n">buffersize</span><span class="o">=</span><span class="mi">4194304</span><span class="p">,</span> <span class="n">blocksize</span><span class="o">=</span><span class="mi">4194304</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Upload a file to Azure Data Lake.</span>
+
+<span class="sd"> :param local_path: local path. Can be single file, directory (in which case,</span>
+<span class="sd"> upload recursively) or glob pattern. Recursive glob patterns using `**`</span>
+<span class="sd"> are not supported.</span>
+<span class="sd"> :type local_path: str</span>
+<span class="sd"> :param remote_path: Remote path to upload to; if multiple files, this is the</span>
+<span class="sd"> dircetory root to write within.</span>
+<span class="sd"> :type remote_path: str</span>
+<span class="sd"> :param nthreads: Number of threads to use. If None, uses the number of cores.</span>
+<span class="sd"> :type nthreads: int</span>
+<span class="sd"> :param overwrite: Whether to forcibly overwrite existing files/directories.</span>
+<span class="sd"> If False and remote path is a directory, will quit regardless if any files</span>
+<span class="sd"> would be overwritten or not. If True, only matching filenames are actually</span>
+<span class="sd"> overwritten.</span>
+<span class="sd"> :type overwrite: bool</span>
+<span class="sd"> :param buffersize: int [2**22]</span>
+<span class="sd"> Number of bytes for internal buffer. This block cannot be bigger than</span>
+<span class="sd"> a chunk and cannot be smaller than a block.</span>
+<span class="sd"> :type buffersize: int</span>
+<span class="sd"> :param blocksize: int [2**22]</span>
+<span class="sd"> Number of bytes for a block. Within each chunk, we write a smaller</span>
+<span class="sd"> block for each API call. This block cannot be bigger than a chunk.</span>
+<span class="sd"> :type blocksize: int</span>
+<span class="sd"> """</span>
+ <span class="n">multithread</span><span class="o">.</span><span class="n">ADLUploader</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="p">,</span>
+ <span class="n">lpath</span><span class="o">=</span><span class="n">local_path</span><span class="p">,</span>
+ <span class="n">rpath</span><span class="o">=</span><span class="n">remote_path</span><span class="p">,</span>
+ <span class="n">nthreads</span><span class="o">=</span><span class="n">nthreads</span><span class="p">,</span>
+ <span class="n">overwrite</span><span class="o">=</span><span class="n">overwrite</span><span class="p">,</span>
+ <span class="n">buffersize</span><span class="o">=</span><span class="n">buffersize</span><span class="p">,</span>
+ <span class="n">blocksize</span><span class="o">=</span><span class="n">blocksize</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureDataLakeHook.download_file"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_data_lake_hook.AzureDataLakeHook.download_file">[docs]</a> <span class="k">def</span> <span class="nf">download_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">local_path</span><span class="p">,</span> <span class="n">remote_path</span><span class="p">,</span> <span class="n">nth [...]
+ <span class="n">buffersize</span><span class="o">=</span><span class="mi">4194304</span><span class="p">,</span> <span class="n">blocksize</span><span class="o">=</span><span class="mi">4194304</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Download a file from Azure Blob Storage.</span>
+
+<span class="sd"> :param local_path: local path. If downloading a single file, will write to this</span>
+<span class="sd"> specific file, unless it is an existing directory, in which case a file is</span>
+<span class="sd"> created within it. If downloading multiple files, this is the root</span>
+<span class="sd"> directory to write within. Will create directories as required.</span>
+<span class="sd"> :type local_path: str</span>
+<span class="sd"> :param remote_path: remote path/globstring to use to find remote files.</span>
+<span class="sd"> Recursive glob patterns using `**` are not supported.</span>
+<span class="sd"> :type remote_path: str</span>
+<span class="sd"> :param nthreads: Number of threads to use. If None, uses the number of cores.</span>
+<span class="sd"> :type nthreads: int</span>
+<span class="sd"> :param overwrite: Whether to forcibly overwrite existing files/directories.</span>
+<span class="sd"> If False and remote path is a directory, will quit regardless if any files</span>
+<span class="sd"> would be overwritten or not. If True, only matching filenames are actually</span>
+<span class="sd"> overwritten.</span>
+<span class="sd"> :type overwrite: bool</span>
+<span class="sd"> :param buffersize: int [2**22]</span>
+<span class="sd"> Number of bytes for internal buffer. This block cannot be bigger than</span>
+<span class="sd"> a chunk and cannot be smaller than a block.</span>
+<span class="sd"> :type buffersize: int</span>
+<span class="sd"> :param blocksize: int [2**22]</span>
+<span class="sd"> Number of bytes for a block. Within each chunk, we write a smaller</span>
+<span class="sd"> block for each API call. This block cannot be bigger than a chunk.</span>
+<span class="sd"> :type blocksize: int</span>
+<span class="sd"> """</span>
+ <span class="n">multithread</span><span class="o">.</span><span class="n">ADLDownloader</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="p">,</span>
+ <span class="n">lpath</span><span class="o">=</span><span class="n">local_path</span><span class="p">,</span>
+ <span class="n">rpath</span><span class="o">=</span><span class="n">remote_path</span><span class="p">,</span>
+ <span class="n">nthreads</span><span class="o">=</span><span class="n">nthreads</span><span class="p">,</span>
+ <span class="n">overwrite</span><span class="o">=</span><span class="n">overwrite</span><span class="p">,</span>
+ <span class="n">buffersize</span><span class="o">=</span><span class="n">buffersize</span><span class="p">,</span>
+ <span class="n">blocksize</span><span class="o">=</span><span class="n">blocksize</span><span class="p">)</span></div></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ LANGUAGE:'None',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/azure_fileshare_hook.html b/_modules/airflow/contrib/hooks/azure_fileshare_hook.html
new file mode 100644
index 0000000..a34fc55
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/azure_fileshare_hook.html
@@ -0,0 +1,430 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.azure_fileshare_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.azure_fileshare_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.azure_fileshare_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">azure.storage.file</span> <span class="k">import</span> <span class="n">FileService</span>
+
+
+<div class="viewcode-block" id="AzureFileShareHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook">[docs]</a><span class="k">class</span> <span class="nc">AzureFileShareHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interacts with Azure FileShare Storage.</span>
+
+<span class="sd"> Additional options passed in the 'extra' field of the connection will be</span>
+<span class="sd"> passed to the `FileService()` constructor.</span>
+
+<span class="sd"> :param wasb_conn_id: Reference to the wasb connection.</span>
+<span class="sd"> :type wasb_conn_id: str</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">wasb_conn_id</span><span class="o">=</span><span class="s1">'wasb_default'</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span> <span class="o">=</span> <span class="n">wasb_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">connection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+<div class="viewcode-block" id="AzureFileShareHook.get_conn"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""Return the FileService object."""</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+ <span class="n">service_options</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span>
+ <span class="k">return</span> <span class="n">FileService</span><span class="p">(</span><span class="n">account_name</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span>
+ <span class="n">account_key</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span> <span class="o">**</span><span class="n">service_options</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.check_for_directory"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.check_for_directory">[docs]</a> <span class="k">def</span> <span class="nf">check_for_directory</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</sp [...]
+ <span class="sd">"""</span>
+<span class="sd"> Check if a directory exists on Azure File Share.</span>
+
+<span class="sd"> :param share_name: Name of the share.</span>
+<span class="sd"> :type share_name: str</span>
+<span class="sd"> :param directory_name: Name of the directory.</span>
+<span class="sd"> :type directory_name: str</span>
+<span class="sd"> :param kwargs: Optional keyword arguments that</span>
+<span class="sd"> `FileService.exists()` takes.</span>
+<span class="sd"> :type kwargs: object</span>
+<span class="sd"> :return: True if the file exists, False otherwise.</span>
+<span class="sd"> :rtype bool</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.check_for_file"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.check_for_file">[docs]</a> <span class="k">def</span> <span class="nf">check_for_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span> <span class [...]
+ <span class="sd">"""</span>
+<span class="sd"> Check if a file exists on Azure File Share.</span>
+
+<span class="sd"> :param share_name: Name of the share.</span>
+<span class="sd"> :type share_name: str</span>
+<span class="sd"> :param directory_name: Name of the directory.</span>
+<span class="sd"> :type directory_name: str</span>
+<span class="sd"> :param file_name: Name of the file.</span>
+<span class="sd"> :type file_name: str</span>
+<span class="sd"> :param kwargs: Optional keyword arguments that</span>
+<span class="sd"> `FileService.exists()` takes.</span>
+<span class="sd"> :type kwargs: object</span>
+<span class="sd"> :return: True if the file exists, False otherwise.</span>
+<span class="sd"> :rtype bool</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span>
+ <span class="n">file_name</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.list_directories_and_files"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.list_directories_and_files">[docs]</a> <span class="k">def</span> <span class="nf">list_directories_and_files</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span> [...]
+ <span class="sd">"""</span>
+<span class="sd"> Return the list of directories and files stored on a Azure File Share.</span>
+
+<span class="sd"> :param share_name: Name of the share.</span>
+<span class="sd"> :type share_name: str</span>
+<span class="sd"> :param directory_name: Name of the directory.</span>
+<span class="sd"> :type directory_name: str</span>
+<span class="sd"> :param kwargs: Optional keyword arguments that</span>
+<span class="sd"> `FileService.list_directories_and_files()` takes.</span>
+<span class="sd"> :type kwargs: object</span>
+<span class="sd"> :return: A list of files and directories</span>
+<span class="sd"> :rtype list</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">list_directories_and_files</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span>
+ <span class="n">directory_name</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.create_directory"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.create_directory">[docs]</a> <span class="k">def</span> <span class="nf">create_directory</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span> <span [...]
+ <span class="sd">"""</span>
+<span class="sd"> Create a new direcotry on a Azure File Share.</span>
+
+<span class="sd"> :param share_name: Name of the share.</span>
+<span class="sd"> :type share_name: str</span>
+<span class="sd"> :param directory_name: Name of the directory.</span>
+<span class="sd"> :type directory_name: str</span>
+<span class="sd"> :param kwargs: Optional keyword arguments that</span>
+<span class="sd"> `FileService.create_directory()` takes.</span>
+<span class="sd"> :type kwargs: object</span>
+<span class="sd"> :return: A list of files and directories</span>
+<span class="sd"> :rtype list</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">create_directory</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.get_file"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.get_file">[docs]</a> <span class="k">def</span> <span class="nf">get_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">file_path</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</sp [...]
+ <span class="sd">"""</span>
+<span class="sd"> Download a file from Azure File Share.</span>
+
+<span class="sd"> :param file_path: Where to store the file.</span>
+<span class="sd"> :type file_path: str</span>
+<span class="sd"> :param share_name: Name of the share.</span>
+<span class="sd"> :type share_name: str</span>
+<span class="sd"> :param directory_name: Name of the directory.</span>
+<span class="sd"> :type directory_name: str</span>
+<span class="sd"> :param file_name: Name of the file.</span>
+<span class="sd"> :type file_name: str</span>
+<span class="sd"> :param kwargs: Optional keyword arguments that</span>
+<span class="sd"> `FileService.get_file_to_path()` takes.</span>
+<span class="sd"> :type kwargs: object</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">get_file_to_path</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span>
+ <span class="n">file_name</span><span class="p">,</span> <span class="n">file_path</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.get_file_to_stream"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.get_file_to_stream">[docs]</a> <span class="k">def</span> <span class="nf">get_file_to_stream</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">stream</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span c [...]
+ <span class="sd">"""</span>
+<span class="sd"> Download a file from Azure File Share.</span>
+
+<span class="sd"> :param stream: A filehandle to store the file to.</span>
+<span class="sd"> :type stream: file-like object</span>
+<span class="sd"> :param share_name: Name of the share.</span>
+<span class="sd"> :type share_name: str</span>
+<span class="sd"> :param directory_name: Name of the directory.</span>
+<span class="sd"> :type directory_name: str</span>
+<span class="sd"> :param file_name: Name of the file.</span>
+<span class="sd"> :type file_name: str</span>
+<span class="sd"> :param kwargs: Optional keyword arguments that</span>
+<span class="sd"> `FileService.get_file_to_stream()` takes.</span>
+<span class="sd"> :type kwargs: object</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">get_file_to_stream</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span>
+ <span class="n">file_name</span><span class="p">,</span> <span class="n">stream</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.load_file"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.load_file">[docs]</a> <span class="k">def</span> <span class="nf">load_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">file_path</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name< [...]
+ <span class="sd">"""</span>
+<span class="sd"> Upload a file to Azure File Share.</span>
+
+<span class="sd"> :param file_path: Path to the file to load.</span>
+<span class="sd"> :type file_path: str</span>
+<span class="sd"> :param share_name: Name of the share.</span>
+<span class="sd"> :type share_name: str</span>
+<span class="sd"> :param directory_name: Name of the directory.</span>
+<span class="sd"> :type directory_name: str</span>
+<span class="sd"> :param file_name: Name of the file.</span>
+<span class="sd"> :type file_name: str</span>
+<span class="sd"> :param kwargs: Optional keyword arguments that</span>
+<span class="sd"> `FileService.create_file_from_path()` takes.</span>
+<span class="sd"> :type kwargs: object</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">create_file_from_path</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span>
+ <span class="n">file_name</span><span class="p">,</span> <span class="n">file_path</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.load_string"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.load_string">[docs]</a> <span class="k">def</span> <span class="nf">load_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">string_data</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span class="n">directo [...]
+ <span class="sd">"""</span>
+<span class="sd"> Upload a string to Azure File Share.</span>
+
+<span class="sd"> :param string_data: String to load.</span>
+<span class="sd"> :type string_data: str</span>
+<span class="sd"> :param share_name: Name of the share.</span>
+<span class="sd"> :type share_name: str</span>
+<span class="sd"> :param directory_name: Name of the directory.</span>
+<span class="sd"> :type directory_name: str</span>
+<span class="sd"> :param file_name: Name of the file.</span>
+<span class="sd"> :type file_name: str</span>
+<span class="sd"> :param kwargs: Optional keyword arguments that</span>
+<span class="sd"> `FileService.create_file_from_text()` takes.</span>
+<span class="sd"> :type kwargs: object</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">create_file_from_text</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span>
+ <span class="n">file_name</span><span class="p">,</span> <span class="n">string_data</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.load_stream"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.load_stream">[docs]</a> <span class="k">def</span> <span class="nf">load_stream</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">stream</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span class="n">directory_na [...]
+ <span class="sd">"""</span>
+<span class="sd"> Upload a stream to Azure File Share.</span>
+
+<span class="sd"> :param stream: Opened file/stream to upload as the file content.</span>
+<span class="sd"> :type stream: file-like</span>
+<span class="sd"> :param share_name: Name of the share.</span>
+<span class="sd"> :type share_name: str</span>
+<span class="sd"> :param directory_name: Name of the directory.</span>
+<span class="sd"> :type directory_name: str</span>
+<span class="sd"> :param file_name: Name of the file.</span>
+<span class="sd"> :type file_name: str</span>
+<span class="sd"> :param count: Size of the stream in bytes</span>
+<span class="sd"> :type count: int</span>
+<span class="sd"> :param kwargs: Optional keyword arguments that</span>
+<span class="sd"> `FileService.create_file_from_stream()` takes.</span>
+<span class="sd"> :type kwargs: object</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">create_file_from_stream</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span>
+ <span class="n">file_name</span><span class="p">,</span> <span class="n">stream</span><span class="p">,</span> <span class="n">count</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ LANGUAGE:'None',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/bigquery_hook.html b/_modules/airflow/contrib/hooks/bigquery_hook.html
index aff2ebd..b02ae65 100644
--- a/_modules/airflow/contrib/hooks/bigquery_hook.html
+++ b/_modules/airflow/contrib/hooks/bigquery_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -243,7 +233,8 @@
<span class="sd"> Returns a BigQuery service object.</span>
<span class="sd"> """</span>
<span class="n">http_authorized</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
- <span class="k">return</span> <span class="n">build</span><span class="p">(</span><span class="s1">'bigquery'</span><span class="p">,</span> <span class="s1">'v2'</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">)</span></div>
+ <span class="k">return</span> <span class="n">build</span><span class="p">(</span>
+ <span class="s1">'bigquery'</span><span class="p">,</span> <span class="s1">'v2'</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">,</span> <span class="n">cache_discovery</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
<div class="viewcode-block" id="BigQueryHook.insert_rows"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook.insert_rows">[docs]</a> <span class="k">def</span> <span class="nf">insert_rows</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">rows</span><span class="p">,</span> <span class="n">target_fields</span><span class="o">=< [...]
<span class="sd">"""</span>
@@ -253,7 +244,7 @@
<span class="sd"> """</span>
<span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">()</span></div>
-<div class="viewcode-block" id="BigQueryHook.get_pandas_df"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook.get_pandas_df">[docs]</a> <span class="k">def</span> <span class="nf">get_pandas_df</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">, [...]
+<div class="viewcode-block" id="BigQueryHook.get_pandas_df"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook.get_pandas_df">[docs]</a> <span class="k">def</span> <span class="nf">get_pandas_df</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">, [...]
<span class="sd">"""</span>
<span class="sd"> Returns a Pandas DataFrame for the results produced by a BigQuery</span>
<span class="sd"> query. The DbApiHook method must be overridden because Pandas</span>
@@ -262,8 +253,8 @@
<span class="sd"> https://github.com/pydata/pandas/blob/master/pandas/io/sql.py#L447</span>
<span class="sd"> https://github.com/pydata/pandas/issues/6900</span>
-<span class="sd"> :param bql: The BigQuery SQL to execute.</span>
-<span class="sd"> :type bql: string</span>
+<span class="sd"> :param sql: The BigQuery SQL to execute.</span>
+<span class="sd"> :type sql: string</span>
<span class="sd"> :param parameters: The parameters to render the SQL query with (not</span>
<span class="sd"> used, leave to override superclass method)</span>
<span class="sd"> :type parameters: mapping or iterable</span>
@@ -274,7 +265,7 @@
<span class="k">if</span> <span class="n">dialect</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">dialect</span> <span class="o">=</span> <span class="s1">'legacy'</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span> <span class="k">else</span> <span class="s1">'standard'</span>
- <span class="k">return</span> <span class="n">read_gbq</span><span class="p">(</span><span class="n">bql</span><span class="p">,</span>
+ <span class="k">return</span> <span class="n">read_gbq</span><span class="p">(</span><span class="n">sql</span><span class="p">,</span>
<span class="n">project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_get_field</span><span class="p">(</span><span class="s1">'project'</span><span class="p">),</span>
<span class="n">dialect</span><span class="o">=</span><span class="n">dialect</span><span class="p">,</span>
<span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
@@ -625,7 +616,8 @@
<span class="p">)</span>
<span class="k">def</span> <span class="nf">run_query</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
- <span class="n">bql</span><span class="p">,</span>
+ <span class="n">bql</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">sql</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">destination_dataset_table</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">write_disposition</span><span class="o">=</span><span class="s1">'WRITE_EMPTY'</span><span class="p">,</span>
<span class="n">allow_large_results</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
@@ -637,7 +629,8 @@
<span class="n">create_disposition</span><span class="o">=</span><span class="s1">'CREATE_IF_NEEDED'</span><span class="p">,</span>
<span class="n">query_params</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">schema_update_options</span><span class="o">=</span><span class="p">(),</span>
- <span class="n">priority</span><span class="o">=</span><span class="s1">'INTERACTIVE'</span><span class="p">):</span>
+ <span class="n">priority</span><span class="o">=</span><span class="s1">'INTERACTIVE'</span><span class="p">,</span>
+ <span class="n">time_partitioning</span><span class="o">=</span><span class="p">{}):</span>
<span class="sd">"""</span>
<span class="sd"> Executes a BigQuery SQL query. Optionally persists results in a BigQuery</span>
<span class="sd"> table. See here:</span>
@@ -646,10 +639,14 @@
<span class="sd"> For more details about these parameters.</span>
-<span class="sd"> :param bql: The BigQuery SQL to execute.</span>
+<span class="sd"> :param bql: (Deprecated. Use `sql` parameter instead) The BigQuery SQL</span>
+<span class="sd"> to execute.</span>
<span class="sd"> :type bql: string</span>
+<span class="sd"> :param sql: The BigQuery SQL to execute.</span>
+<span class="sd"> :type sql: string</span>
<span class="sd"> :param destination_dataset_table: The dotted <dataset>.<table></span>
<span class="sd"> BigQuery table to save the query results.</span>
+<span class="sd"> :type destination_dataset_table: string</span>
<span class="sd"> :param write_disposition: What to do if the table already exists in</span>
<span class="sd"> BigQuery.</span>
<span class="sd"> :type write_disposition: string</span>
@@ -687,8 +684,31 @@
<span class="sd"> Possible values include INTERACTIVE and BATCH.</span>
<span class="sd"> The default value is INTERACTIVE.</span>
<span class="sd"> :type priority: string</span>
+<span class="sd"> :param time_partitioning: configure optional time partitioning fields i.e.</span>
+<span class="sd"> partition by field, type and</span>
+<span class="sd"> expiration as per API specifications. Note that 'field' is not available in</span>
+<span class="sd"> conjunction with dataset.table$partition.</span>
+<span class="sd"> :type time_partitioning: dict</span>
+
<span class="sd"> """</span>
+ <span class="c1"># TODO remove `bql` in Airflow 2.0 - Jira: [AIRFLOW-2513]</span>
+ <span class="n">sql</span> <span class="o">=</span> <span class="n">bql</span> <span class="k">if</span> <span class="n">sql</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">sql</span>
+
+ <span class="k">if</span> <span class="n">bql</span><span class="p">:</span>
+ <span class="kn">import</span> <span class="nn">warnings</span>
+ <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s1">'Deprecated parameter `bql` used in '</span>
+ <span class="s1">'`BigQueryBaseCursor.run_query` '</span>
+ <span class="s1">'Use `sql` parameter instead to pass the sql to be '</span>
+ <span class="s1">'executed. `bql` parameter is deprecated and '</span>
+ <span class="s1">'will be removed in a future version of '</span>
+ <span class="s1">'Airflow.'</span><span class="p">,</span>
+ <span class="n">category</span><span class="o">=</span><span class="ne">DeprecationWarning</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">sql</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">'`BigQueryBaseCursor.run_query` missing 1 required '</span>
+ <span class="s1">'positional argument: `sql`'</span><span class="p">)</span>
+
<span class="c1"># BigQuery also allows you to define how you want a table's schema to change</span>
<span class="c1"># as a side effect of a query job</span>
<span class="c1"># for more details:</span>
@@ -708,7 +728,7 @@
<span class="n">configuration</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">'query'</span><span class="p">:</span> <span class="p">{</span>
- <span class="s1">'query'</span><span class="p">:</span> <span class="n">bql</span><span class="p">,</span>
+ <span class="s1">'query'</span><span class="p">:</span> <span class="n">sql</span><span class="p">,</span>
<span class="s1">'useLegacySql'</span><span class="p">:</span> <span class="n">use_legacy_sql</span><span class="p">,</span>
<span class="s1">'maximumBillingTier'</span><span class="p">:</span> <span class="n">maximum_billing_tier</span><span class="p">,</span>
<span class="s1">'maximumBytesBilled'</span><span class="p">:</span> <span class="n">maximum_bytes_billed</span><span class="p">,</span>
@@ -724,14 +744,10 @@
<span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">destination_dataset_table</span><span class="p">,</span>
<span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">)</span>
<span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">({</span>
- <span class="s1">'allowLargeResults'</span><span class="p">:</span>
- <span class="n">allow_large_results</span><span class="p">,</span>
- <span class="s1">'flattenResults'</span><span class="p">:</span>
- <span class="n">flatten_results</span><span class="p">,</span>
- <span class="s1">'writeDisposition'</span><span class="p">:</span>
- <span class="n">write_disposition</span><span class="p">,</span>
- <span class="s1">'createDisposition'</span><span class="p">:</span>
- <span class="n">create_disposition</span><span class="p">,</span>
+ <span class="s1">'allowLargeResults'</span><span class="p">:</span> <span class="n">allow_large_results</span><span class="p">,</span>
+ <span class="s1">'flattenResults'</span><span class="p">:</span> <span class="n">flatten_results</span><span class="p">,</span>
+ <span class="s1">'writeDisposition'</span><span class="p">:</span> <span class="n">write_disposition</span><span class="p">,</span>
+ <span class="s1">'createDisposition'</span><span class="p">:</span> <span class="n">create_disposition</span><span class="p">,</span>
<span class="s1">'destinationTable'</span><span class="p">:</span> <span class="p">{</span>
<span class="s1">'projectId'</span><span class="p">:</span> <span class="n">destination_project</span><span class="p">,</span>
<span class="s1">'datasetId'</span><span class="p">:</span> <span class="n">destination_dataset</span><span class="p">,</span>
@@ -741,8 +757,7 @@
<span class="k">if</span> <span class="n">udf_config</span><span class="p">:</span>
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">udf_config</span><span class="p">,</span> <span class="nb">list</span><span class="p">)</span>
<span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">({</span>
- <span class="s1">'userDefinedFunctionResources'</span><span class="p">:</span>
- <span class="n">udf_config</span>
+ <span class="s1">'userDefinedFunctionResources'</span><span class="p">:</span> <span class="n">udf_config</span>
<span class="p">})</span>
<span class="k">if</span> <span class="n">query_params</span><span class="p">:</span>
@@ -752,6 +767,15 @@
<span class="k">else</span><span class="p">:</span>
<span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">][</span><span class="s1">'queryParameters'</span><span class="p">]</span> <span class="o">=</span> <span class="n">query_params</span>
+ <span class="n">time_partitioning</span> <span class="o">=</span> <span class="n">_cleanse_time_partitioning</span><span class="p">(</span>
+ <span class="n">destination_dataset_table</span><span class="p">,</span>
+ <span class="n">time_partitioning</span>
+ <span class="p">)</span>
+ <span class="k">if</span> <span class="n">time_partitioning</span><span class="p">:</span>
+ <span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">({</span>
+ <span class="s1">'timePartitioning'</span><span class="p">:</span> <span class="n">time_partitioning</span>
+ <span class="p">})</span>
+
<span class="k">if</span> <span class="n">schema_update_options</span><span class="p">:</span>
<span class="k">if</span> <span class="n">write_disposition</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">"WRITE_APPEND"</span><span class="p">,</span> <span class="s2">"WRITE_TRUNCATE"</span><span class="p">]:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"schema_update_options is only "</span>
@@ -971,7 +995,7 @@
<span class="sd"> :param time_partitioning: configure optional time partitioning fields i.e.</span>
<span class="sd"> partition by field, type and</span>
<span class="sd"> expiration as per API specifications. Note that 'field' is not available in</span>
-<span class="sd"> concurrency with dataset.table$partition.</span>
+<span class="sd"> conjunction with dataset.table$partition.</span>
<span class="sd"> :type time_partitioning: dict</span>
<span class="sd"> """</span>
@@ -1024,21 +1048,14 @@
<span class="p">}</span>
<span class="p">}</span>
- <span class="c1"># if it is a partitioned table ($ is in the table name) add partition load option</span>
- <span class="k">if</span> <span class="s1">'$'</span> <span class="ow">in</span> <span class="n">destination_project_dataset_table</span><span class="p">:</span>
- <span class="k">if</span> <span class="n">time_partitioning</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'field'</span><span class="p">):</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
- <span class="s2">"Cannot specify field partition and partition name "</span>
- <span class="s2">"(dataset.table$partition) at the same time"</span>
- <span class="p">)</span>
- <span class="n">configuration</span><span class="p">[</span><span class="s1">'load'</span><span class="p">][</span><span class="s1">'timePartitioning'</span><span class="p">]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="nb">type</span><span class="o">=</span><span class="s1">'DAY'</span><span class="p">)</span>
-
- <span class="c1"># can specify custom time partitioning options based on a field, or adding</span>
- <span class="c1"># expiration</span>
+ <span class="n">time_partitioning</span> <span class="o">=</span> <span class="n">_cleanse_time_partitioning</span><span class="p">(</span>
+ <span class="n">destination_project_dataset_table</span><span class="p">,</span>
+ <span class="n">time_partitioning</span>
+ <span class="p">)</span>
<span class="k">if</span> <span class="n">time_partitioning</span><span class="p">:</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'load'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'timePartitioning'</span><span class="p">):</span>
- <span class="n">configuration</span><span class="p">[</span><span class="s1">'load'</span><span class="p">][</span><span class="s1">'timePartitioning'</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
- <span class="n">configuration</span><span class="p">[</span><span class="s1">'load'</span><span class="p">][</span><span class="s1">'timePartitioning'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">time_partitioning</span><span class="p">)</span>
+ <span class="n">configuration</span><span class="p">[</span><span class="s1">'load'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">({</span>
+ <span class="s1">'timePartitioning'</span><span class="p">:</span> <span class="n">time_partitioning</span>
+ <span class="p">})</span>
<span class="k">if</span> <span class="n">schema_fields</span><span class="p">:</span>
<span class="n">configuration</span><span class="p">[</span><span class="s1">'load'</span><span class="p">][</span><span class="s1">'schema'</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'fields'</span><span class="p">:</span> <span class="n">schema_fields</span><span class="p">}</span>
@@ -1443,9 +1460,9 @@
<span class="sd"> :param parameters: Parameters to substitute into the query.</span>
<span class="sd"> :type parameters: dict</span>
<span class="sd"> """</span>
- <span class="n">bql</span> <span class="o">=</span> <span class="n">_bind_parameters</span><span class="p">(</span><span class="n">operation</span><span class="p">,</span>
+ <span class="n">sql</span> <span class="o">=</span> <span class="n">_bind_parameters</span><span class="p">(</span><span class="n">operation</span><span class="p">,</span>
<span class="n">parameters</span><span class="p">)</span> <span class="k">if</span> <span class="n">parameters</span> <span class="k">else</span> <span class="n">operation</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_query</span><span class="p">(</span><span class="n">bql</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_query</span><span class="p">(</span><span class="n">sql</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">executemany</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">operation</span><span class="p">,</span> <span class="n">seq_of_parameters</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -1595,9 +1612,9 @@
<span class="sd"> """</span>
<span class="k">if</span> <span class="n">string_field</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">None</span>
- <span class="k">elif</span> <span class="n">bq_type</span> <span class="o">==</span> <span class="s1">'INTEGER'</span> <span class="ow">or</span> <span class="n">bq_type</span> <span class="o">==</span> <span class="s1">'TIMESTAMP'</span><span class="p">:</span>
+ <span class="k">elif</span> <span class="n">bq_type</span> <span class="o">==</span> <span class="s1">'INTEGER'</span><span class="p">:</span>
<span class="k">return</span> <span class="nb">int</span><span class="p">(</span><span class="n">string_field</span><span class="p">)</span>
- <span class="k">elif</span> <span class="n">bq_type</span> <span class="o">==</span> <span class="s1">'FLOAT'</span><span class="p">:</span>
+ <span class="k">elif</span> <span class="n">bq_type</span> <span class="o">==</span> <span class="s1">'FLOAT'</span> <span class="ow">or</span> <span class="n">bq_type</span> <span class="o">==</span> <span class="s1">'TIMESTAMP'</span><span class="p">:</span>
<span class="k">return</span> <span class="nb">float</span><span class="p">(</span><span class="n">string_field</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">bq_type</span> <span class="o">==</span> <span class="s1">'BOOLEAN'</span><span class="p">:</span>
<span class="k">assert</span> <span class="n">string_field</span> <span class="ow">in</span> <span class="nb">set</span><span class="p">([</span><span class="s1">'true'</span><span class="p">,</span> <span class="s1">'false'</span><span class="p">])</span>
@@ -1662,12 +1679,24 @@
<span class="n">project_id</span> <span class="o">=</span> <span class="n">default_project_id</span>
<span class="k">return</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span>
+
+
+<span class="k">def</span> <span class="nf">_cleanse_time_partitioning</span><span class="p">(</span><span class="n">destination_dataset_table</span><span class="p">,</span> <span class="n">time_partitioning_in</span><span class="p">):</span>
+ <span class="c1"># if it is a partitioned table ($ is in the table name) add partition load option</span>
+ <span class="n">time_partitioning_out</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="k">if</span> <span class="n">destination_dataset_table</span> <span class="ow">and</span> <span class="s1">'$'</span> <span class="ow">in</span> <span class="n">destination_dataset_table</span><span class="p">:</span>
+ <span class="k">assert</span> <span class="ow">not</span> <span class="n">time_partitioning_in</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'field'</span><span class="p">),</span> <span class="p">(</span>
+ <span class="s2">"Cannot specify field partition and partition name "</span>
+ <span class="s2">"(dataset.table$partition) at the same time"</span>
+ <span class="p">)</span>
+ <span class="n">time_partitioning_out</span><span class="p">[</span><span class="s1">'type'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'DAY'</span>
+
+ <span class="n">time_partitioning_out</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">time_partitioning_in</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">time_partitioning_out</span>
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -1679,7 +1708,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -1698,6 +1727,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -1710,19 +1740,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/cassandra_hook.html b/_modules/airflow/contrib/hooks/cassandra_hook.html
new file mode 100644
index 0000000..01142f1
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/cassandra_hook.html
@@ -0,0 +1,400 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.cassandra_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.cassandra_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.cassandra_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">cassandra.cluster</span> <span class="k">import</span> <span class="n">Cluster</span>
+<span class="kn">from</span> <span class="nn">cassandra.policies</span> <span class="k">import</span> <span class="p">(</span><span class="n">RoundRobinPolicy</span><span class="p">,</span> <span class="n">DCAwareRoundRobinPolicy</span><span class="p">,</span>
+ <span class="n">TokenAwarePolicy</span><span class="p">,</span> <span class="n">WhiteListRoundRobinPolicy</span><span class="p">)</span>
+<span class="kn">from</span> <span class="nn">cassandra.auth</span> <span class="k">import</span> <span class="n">PlainTextAuthProvider</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+
+<div class="viewcode-block" id="CassandraHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.cassandra_hook.CassandraHook">[docs]</a><span class="k">class</span> <span class="nc">CassandraHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Hook used to interact with Cassandra</span>
+
+<span class="sd"> Contact points can be specified as a comma-separated string in the 'hosts'</span>
+<span class="sd"> field of the connection.</span>
+
+<span class="sd"> Port can be specified in the port field of the connection.</span>
+
+<span class="sd"> If SSL is enabled in Cassandra, pass in a dict in the extra field as kwargs for</span>
+<span class="sd"> ``ssl.wrap_socket()``. For example:</span>
+<span class="sd"> {</span>
+<span class="sd"> 'ssl_options' : {</span>
+<span class="sd"> 'ca_certs' : PATH_TO_CA_CERTS</span>
+<span class="sd"> }</span>
+<span class="sd"> }</span>
+
+<span class="sd"> Default load balancing policy is RoundRobinPolicy. To specify a different LB policy:</span>
+<span class="sd"> - DCAwareRoundRobinPolicy</span>
+<span class="sd"> {</span>
+<span class="sd"> 'load_balancing_policy': 'DCAwareRoundRobinPolicy',</span>
+<span class="sd"> 'load_balancing_policy_args': {</span>
+<span class="sd"> 'local_dc': LOCAL_DC_NAME, // optional</span>
+<span class="sd"> 'used_hosts_per_remote_dc': SOME_INT_VALUE, // optional</span>
+<span class="sd"> }</span>
+<span class="sd"> }</span>
+<span class="sd"> - WhiteListRoundRobinPolicy</span>
+<span class="sd"> {</span>
+<span class="sd"> 'load_balancing_policy': 'WhiteListRoundRobinPolicy',</span>
+<span class="sd"> 'load_balancing_policy_args': {</span>
+<span class="sd"> 'hosts': ['HOST1', 'HOST2', 'HOST3']</span>
+<span class="sd"> }</span>
+<span class="sd"> }</span>
+<span class="sd"> - TokenAwarePolicy</span>
+<span class="sd"> {</span>
+<span class="sd"> 'load_balancing_policy': 'TokenAwarePolicy',</span>
+<span class="sd"> 'load_balancing_policy_args': {</span>
+<span class="sd"> 'child_load_balancing_policy': CHILD_POLICY_NAME, // optional</span>
+<span class="sd"> 'child_load_balancing_policy_args': { ... } // optional</span>
+<span class="sd"> }</span>
+<span class="sd"> }</span>
+
+<span class="sd"> For details of the Cluster config, see cassandra.cluster.</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cassandra_conn_id</span><span class="o">=</span><span class="s1">'cassandra_default'</span><span class="p">):</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">cassandra_conn_id</span><span class="p">)</span>
+
+ <span class="n">conn_config</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">:</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s1">'contact_points'</span><span class="p">]</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">','</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">:</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s1">'port'</span><span class="p">]</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">:</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s1">'auth_provider'</span><span class="p">]</span> <span class="o">=</span> <span class="n">PlainTextAuthProvider</span><span class="p">(</span>
+ <span class="n">username</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span> <span class="n">password</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">)</span>
+
+ <span class="n">policy_name</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'load_balancing_policy'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="n">policy_args</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'load_balancing_policy_args'</span><span class="p">,</span> <span class="p">{})</span>
+ <span class="n">lb_policy</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_lb_policy</span><span class="p">(</span><span class="n">policy_name</span><span class="p">,</span> <span class="n">policy_args</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">lb_policy</span><span class="p">:</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s1">'load_balancing_policy'</span><span class="p">]</span> <span class="o">=</span> <span class="n">lb_policy</span>
+
+ <span class="n">cql_version</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'cql_version'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">cql_version</span><span class="p">:</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s1">'cql_version'</span><span class="p">]</span> <span class="o">=</span> <span class="n">cql_version</span>
+
+ <span class="n">ssl_options</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'ssl_options'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">ssl_options</span><span class="p">:</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s1">'ssl_options'</span><span class="p">]</span> <span class="o">=</span> <span class="n">ssl_options</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">cluster</span> <span class="o">=</span> <span class="n">Cluster</span><span class="p">(</span><span class="o">**</span><span class="n">conn_config</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">keyspace</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">schema</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">session</span> <span class="o">=</span> <span class="kc">None</span>
+
+<div class="viewcode-block" id="CassandraHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.cassandra_hook.CassandraHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns a cassandra Session object</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">is_shutdown</span><span class="p">:</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">session</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">keyspace</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span></div>
+
+ <span class="k">def</span> <span class="nf">get_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster</span>
+
+<div class="viewcode-block" id="CassandraHook.shutdown_cluster"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.cassandra_hook.CassandraHook.shutdown_cluster">[docs]</a> <span class="k">def</span> <span class="nf">shutdown_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Closes all sessions and connections associated with this Cluster.</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster</span><span class="o">.</span><span class="n">is_shutdown</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cluster</span><span class="o">.</span><span class="n">shutdown</span><span class="p">()</span></div>
+
+ <span class="nd">@staticmethod</span>
+ <span class="k">def</span> <span class="nf">get_lb_policy</span><span class="p">(</span><span class="n">policy_name</span><span class="p">,</span> <span class="n">policy_args</span><span class="p">):</span>
+ <span class="n">policies</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'RoundRobinPolicy'</span><span class="p">:</span> <span class="n">RoundRobinPolicy</span><span class="p">,</span>
+ <span class="s1">'DCAwareRoundRobinPolicy'</span><span class="p">:</span> <span class="n">DCAwareRoundRobinPolicy</span><span class="p">,</span>
+ <span class="s1">'WhiteListRoundRobinPolicy'</span><span class="p">:</span> <span class="n">WhiteListRoundRobinPolicy</span><span class="p">,</span>
+ <span class="s1">'TokenAwarePolicy'</span><span class="p">:</span> <span class="n">TokenAwarePolicy</span><span class="p">,</span>
+ <span class="p">}</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">policies</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">policy_name</span><span class="p">)</span> <span class="ow">or</span> <span class="n">policy_name</span> <span class="o">==</span> <span class="s1">'RoundRobinPolicy'</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">RoundRobinPolicy</span><span class="p">()</span>
+
+ <span class="k">if</span> <span class="n">policy_name</span> <span class="o">==</span> <span class="s1">'DCAwareRoundRobinPolicy'</span><span class="p">:</span>
+ <span class="n">local_dc</span> <span class="o">=</span> <span class="n">policy_args</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'local_dc'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
+ <span class="n">used_hosts_per_remote_dc</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">policy_args</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'used_hosts_per_remote_dc'</span><span class="p">,</span> <span class="mi">0</span><span class="p">))</span>
+ <span class="k">return</span> <span class="n">DCAwareRoundRobinPolicy</span><span class="p">(</span><span class="n">local_dc</span><span class="p">,</span> <span class="n">used_hosts_per_remote_dc</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">policy_name</span> <span class="o">==</span> <span class="s1">'WhiteListRoundRobinPolicy'</span><span class="p">:</span>
+ <span class="n">hosts</span> <span class="o">=</span> <span class="n">policy_args</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'hosts'</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">hosts</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s1">'Hosts must be specified for WhiteListRoundRobinPolicy'</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">WhiteListRoundRobinPolicy</span><span class="p">(</span><span class="n">hosts</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">policy_name</span> <span class="o">==</span> <span class="s1">'TokenAwarePolicy'</span><span class="p">:</span>
+ <span class="n">allowed_child_policies</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'RoundRobinPolicy'</span><span class="p">,</span>
+ <span class="s1">'DCAwareRoundRobinPolicy'</span><span class="p">,</span>
+ <span class="s1">'WhiteListRoundRobinPolicy'</span><span class="p">,)</span>
+ <span class="n">child_policy_name</span> <span class="o">=</span> <span class="n">policy_args</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'child_load_balancing_policy'</span><span class="p">,</span>
+ <span class="s1">'RoundRobinPolicy'</span><span class="p">)</span>
+ <span class="n">child_policy_args</span> <span class="o">=</span> <span class="n">policy_args</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'child_load_balancing_policy_args'</span><span class="p">,</span> <span class="p">{})</span>
+ <span class="k">if</span> <span class="n">child_policy_name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">allowed_child_policies</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">TokenAwarePolicy</span><span class="p">(</span><span class="n">RoundRobinPolicy</span><span class="p">())</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">child_policy</span> <span class="o">=</span> <span class="n">CassandraHook</span><span class="o">.</span><span class="n">get_lb_policy</span><span class="p">(</span><span class="n">child_policy_name</span><span class="p">,</span>
+ <span class="n">child_policy_args</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">TokenAwarePolicy</span><span class="p">(</span><span class="n">child_policy</span><span class="p">)</span>
+
+<div class="viewcode-block" id="CassandraHook.record_exists"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.cassandra_hook.CassandraHook.record_exists">[docs]</a> <span class="k">def</span> <span class="nf">record_exists</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">keys</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Checks if a record exists in Cassandra</span>
+
+<span class="sd"> :param table: Target Cassandra table.</span>
+<span class="sd"> Use dot notation to target a specific keyspace.</span>
+<span class="sd"> :type table: string</span>
+<span class="sd"> :param keys: The keys and their values to check the existence.</span>
+<span class="sd"> :type keys: dict</span>
+<span class="sd"> """</span>
+ <span class="n">keyspace</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="k">if</span> <span class="s1">'.'</span> <span class="ow">in</span> <span class="n">table</span><span class="p">:</span>
+ <span class="n">keyspace</span><span class="p">,</span> <span class="n">table</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+ <span class="n">ks</span> <span class="o">=</span> <span class="s2">" AND "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="s2">"</span><span class="si">{}</span><span class="s2">=%(</span><span class="si">{}</span><span class="s2">)s"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">key</span><span class="p">)</spa [...]
+ <span class="n">cql</span> <span class="o">=</span> <span class="s2">"SELECT * FROM </span><span class="si">{keyspace}</span><span class="s2">.</span><span class="si">{table}</span><span class="s2"> WHERE </span><span class="si">{keys}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">keyspace</span><span class="o">=</span><span class="p">(</span><span class="n">keyspace</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">keyspace</span><span class="p">),</span> <span class="n">table</span><span class="o">=</span><span class="n">table</span><span class="p">,</span> <span class="n">keys</span><span class="o">=</span><span class="n">ks</span><span class="p">)</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">rs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">cql</span><span class="p">,</span> <span class="n">keys</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">rs</span><span class="o">.</span><span class="n">one</span><span class="p">()</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+ <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">False</span></div></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ LANGUAGE:'None',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/cloudant_hook.html b/_modules/airflow/contrib/hooks/cloudant_hook.html
new file mode 100644
index 0000000..5e762ea
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/cloudant_hook.html
@@ -0,0 +1,301 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.cloudant_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.cloudant_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.cloudant_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="k">import</span> <span class="n">unicode</span>
+
+<span class="kn">import</span> <span class="nn">cloudant</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+
+<div class="viewcode-block" id="CloudantHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.cloudant_hook.CloudantHook">[docs]</a><span class="k">class</span> <span class="nc">CloudantHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+ <span class="sd">"""Interact with Cloudant.</span>
+
+<span class="sd"> This class is a thin wrapper around the cloudant python library. See the</span>
+<span class="sd"> documentation `here <https://github.com/cloudant-labs/cloudant-python>`_.</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cloudant_conn_id</span><span class="o">=</span><span class="s1">'cloudant_default'</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">CloudantHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="s1">'cloudant'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cloudant_conn_id</span> <span class="o">=</span> <span class="n">cloudant_conn_id</span>
+
+ <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">_str</span><span class="p">(</span><span class="n">s</span><span class="p">):</span>
+ <span class="c1"># cloudant-python doesn't support unicode.</span>
+ <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="n">unicode</span><span class="p">):</span>
+ <span class="n">log</span> <span class="o">=</span> <span class="n">LoggingMixin</span><span class="p">()</span><span class="o">.</span><span class="n">log</span>
+ <span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
+ <span class="s1">'cloudant-python does not support unicode. Encoding </span><span class="si">%s</span><span class="s1"> as '</span>
+ <span class="s1">'ascii using "ignore".'</span><span class="p">,</span> <span class="n">s</span>
+ <span class="p">)</span>
+ <span class="k">return</span> <span class="n">s</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'ascii'</span><span class="p">,</span> <span class="s1">'ignore'</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">s</span>
+
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cloudant_conn_id</span><span class="p">)</span>
+
+ <span class="k">for</span> <span class="n">conn_param</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'host'</span><span class="p">,</span> <span class="s1">'password'</span><span class="p">,</span> <span class="s1">'schema'</span><span class="p">]:</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">conn</span><span class="p">,</span> <span class="n">conn_param</span><span class="p">)</span> <span class="ow">or</span> <span class="ow">not</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">conn</span><span class="p">,</span> <span class="n">conn_param</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'missing connection parameter </span><span class="si">{0}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">conn_param</span><span class="p">)</span>
+ <span class="p">)</span>
+
+ <span class="c1"># In the connection form:</span>
+ <span class="c1"># - 'host' is renamed to 'Account'</span>
+ <span class="c1"># - 'login' is renamed 'Username (or API Key)'</span>
+ <span class="c1"># - 'schema' is renamed to 'Database'</span>
+ <span class="c1">#</span>
+ <span class="c1"># So, use the 'host' attribute as the account name, and, if login is</span>
+ <span class="c1"># defined, use that as the username.</span>
+ <span class="n">account</span> <span class="o">=</span> <span class="n">cloudant</span><span class="o">.</span><span class="n">Account</span><span class="p">(</span><span class="n">_str</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">))</span>
+
+ <span class="n">username</span> <span class="o">=</span> <span class="n">_str</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span> <span class="ow">or</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">)</span>
+
+ <span class="n">account</span><span class="o">.</span><span class="n">login</span><span class="p">(</span>
+ <span class="n">username</span><span class="p">,</span>
+ <span class="n">_str</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">))</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
+
+ <span class="k">return</span> <span class="n">account</span><span class="o">.</span><span class="n">database</span><span class="p">(</span><span class="n">_str</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">schema</span><span class="p">))</span>
+
+<div class="viewcode-block" id="CloudantHook.db"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.cloudant_hook.CloudantHook.db">[docs]</a> <span class="k">def</span> <span class="nf">db</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""Returns the Database object for this hook.</span>
+
+<span class="sd"> See the documentation for cloudant-python here</span>
+<span class="sd"> https://github.com/cloudant-labs/cloudant-python.</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span></div></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ LANGUAGE:'None',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/databricks_hook.html b/_modules/airflow/contrib/hooks/databricks_hook.html
index 00ae02f..0d21a4a 100644
--- a/_modules/airflow/contrib/hooks/databricks_hook.html
+++ b/_modules/airflow/contrib/hooks/databricks_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -288,7 +278,7 @@
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Unexpected HTTP Method: '</span> <span class="o">+</span> <span class="n">method</span><span class="p">)</span>
- <span class="k">for</span> <span class="n">attempt_num</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">retry_limit</span><span class="o">+</span><span class="mi">1</span><span class="p">):</span>
+ <span class="k">for</span> <span class="n">attempt_num</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">retry_limit</span> <span class="o">+</span> <span class="mi">1</span><span class="p">):</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">response</span> <span class="o">=</span> <span class="n">request_func</span><span class="p">(</span>
<span class="n">url</span><span class="p">,</span>
@@ -366,10 +356,11 @@
<span class="nd">@property</span>
<span class="k">def</span> <span class="nf">is_terminal</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">life_cycle_state</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">RUN_LIFE_CYCLE_STATES</span><span class="p">:</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">((</span><span class="s1">'Unexpected life cycle state: </span><span class="si">{}</span><span class="s1">: If the state has '</span>
- <span class="s1">'been introduced recently, please check the Databricks user '</span>
- <span class="s1">'guide for troubleshooting information'</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">life_cycle_state</span><span class="p">))</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="p">(</span><span class="s1">'Unexpected life cycle state: </span><span class="si">{}</span><span class="s1">: If the state has '</span>
+ <span class="s1">'been introduced recently, please check the Databricks user '</span>
+ <span class="s1">'guide for troubleshooting information'</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">life_cycle_state</span><span class="p">))</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">life_cycle_state</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'TERMINATED'</span><span class="p">,</span> <span class="s1">'SKIPPED'</span><span class="p">,</span> <span class="s1">'INTERNAL_ERROR'</span><span class="p">)</span>
<span class="nd">@property</span>
@@ -399,9 +390,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -413,7 +402,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -432,6 +421,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -444,19 +434,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/datadog_hook.html b/_modules/airflow/contrib/hooks/datadog_hook.html
index b8c8e04..bb0a05f 100644
--- a/_modules/airflow/contrib/hooks/datadog_hook.html
+++ b/_modules/airflow/contrib/hooks/datadog_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -219,9 +209,11 @@
<span class="bp">self</span><span class="o">.</span><span class="n">host</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">api_key</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"api_key must be specified in the Datadog connection details"</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"api_key must be specified in the "</span>
+ <span class="s2">"Datadog connection details"</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">app_key</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"app_key must be specified in the Datadog connection details"</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"app_key must be specified in the "</span>
+ <span class="s2">"Datadog connection details"</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Setting up api keys for Datadog"</span><span class="p">)</span>
<span class="n">options</span> <span class="o">=</span> <span class="p">{</span>
@@ -260,8 +252,8 @@
<span class="n">from_seconds_ago</span><span class="p">,</span>
<span class="n">to_seconds_ago</span><span class="p">):</span>
<span class="sd">"""</span>
-<span class="sd"> Queries datadog for a specific metric, potentially with some function applied to it</span>
-<span class="sd"> and returns the results.</span>
+<span class="sd"> Queries datadog for a specific metric, potentially with some</span>
+<span class="sd"> function applied to it and returns the results.</span>
<span class="sd"> :param query: The datadog query to execute (see datadog docs)</span>
<span class="sd"> :type query: string</span>
@@ -283,8 +275,8 @@
<div class="viewcode-block" id="DatadogHook.post_event"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.datadog_hook.DatadogHook.post_event">[docs]</a> <span class="k">def</span> <span class="nf">post_event</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">title</span><span class="p">,</span> <span class="n">text</span><span class="p">,</span> <span class="n">tags</span><span class="o">=</span><span class="kc" [...]
<span class="sd">"""</span>
<span class="sd"> Posts an event to datadog (processing finished, potentially alerts, other issues)</span>
-<span class="sd"> Think about this as a means to maintain persistence of alerts, rather than alerting</span>
-<span class="sd"> itself.</span>
+<span class="sd"> Think about this as a means to maintain persistence of alerts, rather than</span>
+<span class="sd"> alerting itself.</span>
<span class="sd"> :param title: The title of the event</span>
<span class="sd"> :type title: string</span>
@@ -312,9 +304,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -326,7 +316,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -345,6 +335,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -357,19 +348,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/datastore_hook.html b/_modules/airflow/contrib/hooks/datastore_hook.html
index 24ae17e..29aa16f 100644
--- a/_modules/airflow/contrib/hooks/datastore_hook.html
+++ b/_modules/airflow/contrib/hooks/datastore_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -189,7 +179,6 @@
<span class="c1"># under the License.</span>
<span class="c1">#</span>
-<span class="kn">import</span> <span class="nn">json</span>
<span class="kn">import</span> <span class="nn">time</span>
<span class="kn">from</span> <span class="nn">apiclient.discovery</span> <span class="k">import</span> <span class="n">build</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="k">import</span> <span class="n">GoogleCloudBaseHook</span>
@@ -216,7 +205,8 @@
<span class="sd"> Returns a Google Cloud Storage service object.</span>
<span class="sd"> """</span>
<span class="n">http_authorized</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
- <span class="k">return</span> <span class="n">build</span><span class="p">(</span><span class="s1">'datastore'</span><span class="p">,</span> <span class="n">version</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">)</span></div>
+ <span class="k">return</span> <span class="n">build</span><span class="p">(</span>
+ <span class="s1">'datastore'</span><span class="p">,</span> <span class="n">version</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">,</span> <span class="n">cache_discovery</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
<div class="viewcode-block" id="DatastoreHook.allocate_ids"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.allocate_ids">[docs]</a> <span class="k">def</span> <span class="nf">allocate_ids</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">partialKeys</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -226,17 +216,22 @@
<span class="sd"> :param partialKeys: a list of partial keys</span>
<span class="sd"> :return: a list of full keys.</span>
<span class="sd"> """</span>
- <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">allocateIds</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">bo [...]
+ <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">allocateIds</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="p">{</span><span class="s1">'keys'</span><span class="p">:</span> <span class="n">partialKeys</span><span class="p">}</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
<span class="k">return</span> <span class="n">resp</span><span class="p">[</span><span class="s1">'keys'</span><span class="p">]</span></div>
<div class="viewcode-block" id="DatastoreHook.begin_transaction"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.begin_transaction">[docs]</a> <span class="k">def</span> <span class="nf">begin_transaction</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Get a new transaction handle</span>
-<span class="sd"> see https://cloud.google.com/datastore/docs/reference/rest/v1/projects/beginTransaction</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> https://cloud.google.com/datastore/docs/reference/rest/v1/projects/beginTransaction</span>
<span class="sd"> :return: a transaction handle</span>
<span class="sd"> """</span>
- <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">beginTransaction</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class=" [...]
+ <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">beginTransaction</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="p">{})</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
<span class="k">return</span> <span class="n">resp</span><span class="p">[</span><span class="s1">'transaction'</span><span class="p">]</span></div>
<div class="viewcode-block" id="DatastoreHook.commit"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.commit">[docs]</a> <span class="k">def</span> <span class="nf">commit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">body</span><span class="p">):</span>
@@ -249,7 +244,8 @@
<span class="sd"> :param body: the body of the commit request</span>
<span class="sd"> :return: the response body of the commit request</span>
<span class="sd"> """</span>
- <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">commit</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</s [...]
+ <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">commit</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
<span class="k">return</span> <span class="n">resp</span></div>
<div class="viewcode-block" id="DatastoreHook.lookup"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.lookup">[docs]</a> <span class="k">def</span> <span class="nf">lookup</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">keys</span><span class="p">,</span> <span class="n">read_consistency</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <sp [...]
@@ -270,7 +266,8 @@
<span class="n">body</span><span class="p">[</span><span class="s1">'readConsistency'</span><span class="p">]</span> <span class="o">=</span> <span class="n">read_consistency</span>
<span class="k">if</span> <span class="n">transaction</span><span class="p">:</span>
<span class="n">body</span><span class="p">[</span><span class="s1">'transaction'</span><span class="p">]</span> <span class="o">=</span> <span class="n">transaction</span>
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">lookup</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</ [...]
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">lookup</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span></div>
<div class="viewcode-block" id="DatastoreHook.rollback"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.rollback">[docs]</a> <span class="k">def</span> <span class="nf">rollback</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">transaction</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -281,7 +278,8 @@
<span class="sd"> :param transaction: the transaction to roll back</span>
<span class="sd"> """</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">rollback</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="p">{</span [...]
+ <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">rollback</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="p">{</span><span class="s1">'transaction'</span><span class="p">:</span> <span class="n">transaction</span><span class="p">})</span>\
<span class="o">.</span><span class="n">execute</span><span class="p">()</span></div>
<div class="viewcode-block" id="DatastoreHook.run_query"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.run_query">[docs]</a> <span class="k">def</span> <span class="nf">run_query</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">body</span><span class="p">):</span>
@@ -294,7 +292,8 @@
<span class="sd"> :param body: the body of the query request</span>
<span class="sd"> :return: the batch of query results.</span>
<span class="sd"> """</span>
- <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">runQuery</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body< [...]
+ <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">runQuery</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
<span class="k">return</span> <span class="n">resp</span><span class="p">[</span><span class="s1">'batch'</span><span class="p">]</span></div>
<div class="viewcode-block" id="DatastoreHook.get_operation"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.get_operation">[docs]</a> <span class="k">def</span> <span class="nf">get_operation</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
@@ -329,7 +328,8 @@
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="n">result</span></div>
-<div class="viewcode-block" id="DatastoreHook.export_to_storage_bucket"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.export_to_storage_bucket">[docs]</a> <span class="k">def</span> <span class="nf">export_to_storage_bucket</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="n">namespace</span><span class="o">=</span><span c [...]
+<div class="viewcode-block" id="DatastoreHook.export_to_storage_bucket"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.export_to_storage_bucket">[docs]</a> <span class="k">def</span> <span class="nf">export_to_storage_bucket</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="n">namespace</span><span class="o">=</span><span c [...]
+ <span class="n">entity_filter</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Export entities from Cloud Datastore to Cloud Storage for backup</span>
<span class="sd"> """</span>
@@ -343,10 +343,12 @@
<span class="s1">'entityFilter'</span><span class="p">:</span> <span class="n">entity_filter</span><span class="p">,</span>
<span class="s1">'labels'</span><span class="p">:</span> <span class="n">labels</span><span class="p">,</span>
<span class="p">}</span>
- <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">admin_connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">export</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">b [...]
+ <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">admin_connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">export</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
<span class="k">return</span> <span class="n">resp</span></div>
-<div class="viewcode-block" id="DatastoreHook.import_from_storage_bucket"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.import_from_storage_bucket">[docs]</a> <span class="k">def</span> <span class="nf">import_from_storage_bucket</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="n">file</span><span class="p">,</span> <span [...]
+<div class="viewcode-block" id="DatastoreHook.import_from_storage_bucket"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.import_from_storage_bucket">[docs]</a> <span class="k">def</span> <span class="nf">import_from_storage_bucket</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="n">file</span><span class="p">,</span>
+ <span class="n">namespace</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">entity_filter</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Import a backup from Cloud Storage to Cloud Datastore</span>
<span class="sd"> """</span>
@@ -360,14 +362,13 @@
<span class="s1">'entityFilter'</span><span class="p">:</span> <span class="n">entity_filter</span><span class="p">,</span>
<span class="s1">'labels'</span><span class="p">:</span> <span class="n">labels</span><span class="p">,</span>
<span class="p">}</span>
- <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">admin_connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">import_</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n"> [...]
+ <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">admin_connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">import_</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
<span class="k">return</span> <span class="n">resp</span></div></div>
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -379,7 +380,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -398,6 +399,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -410,19 +412,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/discord_webhook_hook.html b/_modules/airflow/contrib/hooks/discord_webhook_hook.html
index 1115b8c..b82d631 100644
--- a/_modules/airflow/contrib/hooks/discord_webhook_hook.html
+++ b/_modules/airflow/contrib/hooks/discord_webhook_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -312,9 +302,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -326,7 +314,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -345,6 +333,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -357,19 +346,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/emr_hook.html b/_modules/airflow/contrib/hooks/emr_hook.html
index 8a5a5c7..7aa2a43 100644
--- a/_modules/airflow/contrib/hooks/emr_hook.html
+++ b/_modules/airflow/contrib/hooks/emr_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -194,7 +184,8 @@
<div class="viewcode-block" id="EmrHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.emr_hook.EmrHook">[docs]</a><span class="k">class</span> <span class="nc">EmrHook</span><span class="p">(</span><span class="n">AwsHook</span><span class="p">):</span>
<span class="sd">"""</span>
-<span class="sd"> Interact with AWS EMR. emr_conn_id is only neccessary for using the create_job_flow method.</span>
+<span class="sd"> Interact with AWS EMR. emr_conn_id is only neccessary for using the</span>
+<span class="sd"> create_job_flow method.</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">emr_conn_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
@@ -208,7 +199,8 @@
<div class="viewcode-block" id="EmrHook.create_job_flow"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.emr_hook.EmrHook.create_job_flow">[docs]</a> <span class="k">def</span> <span class="nf">create_job_flow</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">job_flow_overrides</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Creates a job flow using the config from the EMR connection.</span>
-<span class="sd"> Keys of the json extra hash may have the arguments of the boto3 run_job_flow method.</span>
+<span class="sd"> Keys of the json extra hash may have the arguments of the boto3</span>
+<span class="sd"> run_job_flow method.</span>
<span class="sd"> Overrides for this config may be passed as the job_flow_overrides.</span>
<span class="sd"> """</span>
@@ -239,9 +231,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -253,7 +243,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -272,6 +262,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -284,19 +275,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/fs_hook.html b/_modules/airflow/contrib/hooks/fs_hook.html
index 984dd40..7b01482 100644
--- a/_modules/airflow/contrib/hooks/fs_hook.html
+++ b/_modules/airflow/contrib/hooks/fs_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -193,7 +183,7 @@
<div class="viewcode-block" id="FSHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.fs_hook.FSHook">[docs]</a><span class="k">class</span> <span class="nc">FSHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
- <span class="sd">'''</span>
+ <span class="sd">"""</span>
<span class="sd"> Allows for interaction with an file server.</span>
<span class="sd"> Connection should have a name and a path specified under extra:</span>
@@ -203,7 +193,7 @@
<span class="sd"> Conn Type: File (path)</span>
<span class="sd"> Host, Shchema, Login, Password, Port: empty</span>
<span class="sd"> Extra: {"path": "/tmp"}</span>
-<span class="sd"> '''</span>
+<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn_id</span><span class="o">=</span><span class="s1">'fs_default'</span><span class="p">):</span>
<span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">conn_id</span><span class="p">)</span>
@@ -218,9 +208,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -232,7 +220,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -251,6 +239,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -263,19 +252,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/ftp_hook.html b/_modules/airflow/contrib/hooks/ftp_hook.html
index 07b434f..88879d9 100644
--- a/_modules/airflow/contrib/hooks/ftp_hook.html
+++ b/_modules/airflow/contrib/hooks/ftp_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -421,7 +411,7 @@
<span class="n">params</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ftp_conn_id</span><span class="p">)</span>
<span class="k">if</span> <span class="n">params</span><span class="o">.</span><span class="n">port</span><span class="p">:</span>
- <span class="n">ftplib</span><span class="o">.</span><span class="n">FTP_TLS</span><span class="o">.</span><span class="n">port</span><span class="o">=</span><span class="n">params</span><span class="o">.</span><span class="n">port</span>
+ <span class="n">ftplib</span><span class="o">.</span><span class="n">FTP_TLS</span><span class="o">.</span><span class="n">port</span> <span class="o">=</span> <span class="n">params</span><span class="o">.</span><span class="n">port</span>
<span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="n">ftplib</span><span class="o">.</span><span class="n">FTP_TLS</span><span class="p">(</span>
<span class="n">params</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> <span class="n">params</span><span class="o">.</span><span class="n">login</span><span class="p">,</span> <span class="n">params</span><span class="o">.</span><span class="n">password</span>
@@ -431,9 +421,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -445,7 +433,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -464,6 +452,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -476,19 +465,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/gcp_api_base_hook.html b/_modules/airflow/contrib/hooks/gcp_api_base_hook.html
index c81584e..78b729d 100644
--- a/_modules/airflow/contrib/hooks/gcp_api_base_hook.html
+++ b/_modules/airflow/contrib/hooks/gcp_api_base_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -191,14 +181,18 @@
<span class="kn">import</span> <span class="nn">json</span>
<span class="kn">import</span> <span class="nn">httplib2</span>
-<span class="kn">from</span> <span class="nn">oauth2client.client</span> <span class="k">import</span> <span class="n">GoogleCredentials</span>
-<span class="kn">from</span> <span class="nn">oauth2client.service_account</span> <span class="k">import</span> <span class="n">ServiceAccountCredentials</span>
+<span class="kn">import</span> <span class="nn">google.auth</span>
+<span class="kn">import</span> <span class="nn">google_auth_httplib2</span>
+<span class="kn">import</span> <span class="nn">google.oauth2.service_account</span>
<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+<span class="n">_DEFAULT_SCOPES</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'https://www.googleapis.com/auth/cloud-platform'</span><span class="p">,)</span>
+
+
<div class="viewcode-block" id="GoogleCloudBaseHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook">[docs]</a><span class="k">class</span> <span class="nc">GoogleCloudBaseHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> A base hook for Google cloud-related hooks. Google cloud has a shared REST</span>
@@ -212,8 +206,10 @@
<span class="sd"> All hook derived from this base hook use the 'Google Cloud Platform' connection</span>
<span class="sd"> type. Two ways of authentication are supported:</span>
-<span class="sd"> Default credentials: Only specify 'Project Id'. Then you need to have executed</span>
-<span class="sd"> ``gcloud auth`` on the Airflow worker machine.</span>
+<span class="sd"> Default credentials: Only the 'Project Id' is required. You'll need to</span>
+<span class="sd"> have set up default credentials, such as by the</span>
+<span class="sd"> ``GOOGLE_APPLICATION_DEFAULT`` environment variable or from the metadata</span>
+<span class="sd"> server on Google Compute Engine.</span>
<span class="sd"> JSON key file: Specify 'Project Id', 'Key Path' and 'Scope'.</span>
@@ -239,32 +235,30 @@
<span class="sd"> """</span>
<span class="n">key_path</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_field</span><span class="p">(</span><span class="s1">'key_path'</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
<span class="n">keyfile_dict</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_field</span><span class="p">(</span><span class="s1">'keyfile_dict'</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
- <span class="n">scope</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_field</span><span class="p">(</span><span class="s1">'scope'</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
+ <span class="n">scope</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_field</span><span class="p">(</span><span class="s1">'scope'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">scope</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">scopes</span> <span class="o">=</span> <span class="p">[</span><span class="n">s</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">scope</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">','</span><span class="p">)]</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">scopes</span> <span class="o">=</span> <span class="n">_DEFAULT_SCOPES</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">key_path</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">keyfile_dict</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Getting connection using `gcloud auth` user, '</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Getting connection using `google.auth.default()` '</span>
<span class="s1">'since no key file is defined for hook.'</span><span class="p">)</span>
- <span class="n">credentials</span> <span class="o">=</span> <span class="n">GoogleCredentials</span><span class="o">.</span><span class="n">get_application_default</span><span class="p">()</span>
+ <span class="n">credentials</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">google</span><span class="o">.</span><span class="n">auth</span><span class="o">.</span><span class="n">default</span><span class="p">(</span><span class="n">scopes</span><span class="o">=</span><span class="n">scopes</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">key_path</span><span class="p">:</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">scope</span><span class="p">:</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Scope should be defined when using a key file.'</span><span class="p">)</span>
- <span class="n">scopes</span> <span class="o">=</span> <span class="p">[</span><span class="n">s</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">scope</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">','</span><span class="p">)]</span>
-
<span class="c1"># Get credentials from a JSON file.</span>
<span class="k">if</span> <span class="n">key_path</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s1">'.json'</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Getting connection using a JSON key file.'</span><span class="p">)</span>
- <span class="n">credentials</span> <span class="o">=</span> <span class="n">ServiceAccountCredentials</span> \
- <span class="o">.</span><span class="n">from_json_keyfile_name</span><span class="p">(</span><span class="n">key_path</span><span class="p">,</span> <span class="n">scopes</span><span class="p">)</span>
+ <span class="n">credentials</span> <span class="o">=</span> <span class="p">(</span>
+ <span class="n">google</span><span class="o">.</span><span class="n">oauth2</span><span class="o">.</span><span class="n">service_account</span><span class="o">.</span><span class="n">Credentials</span><span class="o">.</span><span class="n">from_service_account_file</span><span class="p">(</span>
+ <span class="n">key_path</span><span class="p">,</span> <span class="n">scopes</span><span class="o">=</span><span class="n">scopes</span><span class="p">)</span>
+ <span class="p">)</span>
<span class="k">elif</span> <span class="n">key_path</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s1">'.p12'</span><span class="p">):</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Legacy P12 key file are not supported, '</span>
<span class="s1">'use a JSON key file.'</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Unrecognised extension for key file.'</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">scope</span><span class="p">:</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Scope should be defined when using key JSON.'</span><span class="p">)</span>
- <span class="n">scopes</span> <span class="o">=</span> <span class="p">[</span><span class="n">s</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">scope</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">','</span><span class="p">)]</span>
-
<span class="c1"># Get credentials from JSON data provided in the UI.</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">keyfile_dict</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">keyfile_dict</span><span class="p">)</span>
@@ -274,19 +268,21 @@
<span class="n">keyfile_dict</span><span class="p">[</span><span class="s1">'private_key'</span><span class="p">]</span> <span class="o">=</span> <span class="n">keyfile_dict</span><span class="p">[</span><span class="s1">'private_key'</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span>
<span class="s1">'</span><span class="se">\\</span><span class="s1">n'</span><span class="p">,</span> <span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">)</span>
- <span class="n">credentials</span> <span class="o">=</span> <span class="n">ServiceAccountCredentials</span> \
- <span class="o">.</span><span class="n">from_json_keyfile_dict</span><span class="p">(</span><span class="n">keyfile_dict</span><span class="p">,</span> <span class="n">scopes</span><span class="p">)</span>
+ <span class="n">credentials</span> <span class="o">=</span> <span class="p">(</span>
+ <span class="n">google</span><span class="o">.</span><span class="n">oauth2</span><span class="o">.</span><span class="n">service_account</span><span class="o">.</span><span class="n">Credentials</span><span class="o">.</span><span class="n">from_service_account_info</span><span class="p">(</span>
+ <span class="n">keyfile_dict</span><span class="p">,</span> <span class="n">scopes</span><span class="o">=</span><span class="n">scopes</span><span class="p">)</span>
+ <span class="p">)</span>
<span class="k">except</span> <span class="n">json</span><span class="o">.</span><span class="n">decoder</span><span class="o">.</span><span class="n">JSONDecodeError</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Invalid key JSON.'</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">credentials</span><span class="o">.</span><span class="n">create_delegated</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span> \
+ <span class="k">return</span> <span class="n">credentials</span><span class="o">.</span><span class="n">with_subject</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span> \
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="k">else</span> <span class="n">credentials</span>
<span class="k">def</span> <span class="nf">_get_access_token</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Returns a valid access token from Google API Credentials</span>
<span class="sd"> """</span>
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_credentials</span><span class="p">()</span><span class="o">.</span><span class="n">get_access_token</span><span class="p">()</span><span class="o">.</span><span class="n">access_token</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_credentials</span><span class="p">()</span><span class="o">.</span><span class="n">token</span>
<span class="k">def</span> <span class="nf">_authorize</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -295,7 +291,9 @@
<span class="sd"> """</span>
<span class="n">credentials</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_credentials</span><span class="p">()</span>
<span class="n">http</span> <span class="o">=</span> <span class="n">httplib2</span><span class="o">.</span><span class="n">Http</span><span class="p">()</span>
- <span class="k">return</span> <span class="n">credentials</span><span class="o">.</span><span class="n">authorize</span><span class="p">(</span><span class="n">http</span><span class="p">)</span>
+ <span class="n">authed_http</span> <span class="o">=</span> <span class="n">google_auth_httplib2</span><span class="o">.</span><span class="n">AuthorizedHttp</span><span class="p">(</span>
+ <span class="n">credentials</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">authed_http</span>
<span class="k">def</span> <span class="nf">_get_field</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">f</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -316,9 +314,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -330,7 +326,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -349,6 +345,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -361,19 +358,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/gcp_container_hook.html b/_modules/airflow/contrib/hooks/gcp_container_hook.html
new file mode 100644
index 0000000..4cb7114
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/gcp_container_hook.html
@@ -0,0 +1,435 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.gcp_container_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.gcp_container_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.gcp_container_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+<span class="kn">import</span> <span class="nn">json</span>
+<span class="kn">import</span> <span class="nn">time</span>
+
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">AirflowException</span><span class="p">,</span> <span class="n">version</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+
+<span class="kn">from</span> <span class="nn">google.api_core.exceptions</span> <span class="k">import</span> <span class="n">AlreadyExists</span>
+<span class="kn">from</span> <span class="nn">google.api_core.gapic_v1.method</span> <span class="k">import</span> <span class="n">DEFAULT</span>
+<span class="kn">from</span> <span class="nn">google.cloud</span> <span class="k">import</span> <span class="n">container_v1</span><span class="p">,</span> <span class="n">exceptions</span>
+<span class="kn">from</span> <span class="nn">google.cloud.container_v1.gapic.enums</span> <span class="k">import</span> <span class="n">Operation</span>
+<span class="kn">from</span> <span class="nn">google.cloud.container_v1.types</span> <span class="k">import</span> <span class="n">Cluster</span>
+<span class="kn">from</span> <span class="nn">google.protobuf</span> <span class="k">import</span> <span class="n">json_format</span>
+<span class="kn">from</span> <span class="nn">google.api_core.gapic_v1.client_info</span> <span class="k">import</span> <span class="n">ClientInfo</span>
+
+<span class="n">OPERATIONAL_POLL_INTERVAL</span> <span class="o">=</span> <span class="mi">15</span>
+
+
+<div class="viewcode-block" id="GKEClusterHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_container_hook.GKEClusterHook">[docs]</a><span class="k">class</span> <span class="nc">GKEClusterHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">location</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">location</span> <span class="o">=</span> <span class="n">location</span>
+
+ <span class="c1"># Add client library info for better error tracking</span>
+ <span class="n">client_info</span> <span class="o">=</span> <span class="n">ClientInfo</span><span class="p">(</span><span class="n">client_library_version</span><span class="o">=</span><span class="s1">'airflow_v'</span> <span class="o">+</span> <span class="n">version</span><span class="o">.</span><span class="n">version</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="n">container_v1</span><span class="o">.</span><span class="n">ClusterManagerClient</span><span class="p">(</span><span class="n">client_info</span><span class="o">=</span><span class="n">client_info</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_dict_to_proto</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">py_dict</span><span class="p">,</span> <span class="n">proto</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Converts a python dictionary to the proto supplied</span>
+<span class="sd"> :param py_dict: The dictionary to convert</span>
+<span class="sd"> :type py_dict: dict</span>
+<span class="sd"> :param proto: The proto object to merge with dictionary</span>
+<span class="sd"> :type proto: protobuf</span>
+<span class="sd"> :return: A parsed python dictionary in provided proto format</span>
+<span class="sd"> :raises:</span>
+<span class="sd"> ParseError: On JSON parsing problems.</span>
+<span class="sd"> """</span>
+ <span class="n">dict_json_str</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">py_dict</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">json_format</span><span class="o">.</span><span class="n">Parse</span><span class="p">(</span><span class="n">dict_json_str</span><span class="p">,</span> <span class="n">proto</span><span class="p">)</span>
+
+<div class="viewcode-block" id="GKEClusterHook.wait_for_operation"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_container_hook.GKEClusterHook.wait_for_operation">[docs]</a> <span class="k">def</span> <span class="nf">wait_for_operation</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">operation</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Given an operation, continuously fetches the status from Google Cloud until either</span>
+<span class="sd"> completion or an error occurring</span>
+<span class="sd"> :param operation: The Operation to wait for</span>
+<span class="sd"> :type operation: A google.cloud.container_V1.gapic.enums.Operator</span>
+<span class="sd"> :return: A new, updated operation fetched from Google Cloud</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Waiting for OPERATION_NAME </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">operation</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
+ <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="n">OPERATIONAL_POLL_INTERVAL</span><span class="p">)</span>
+ <span class="k">while</span> <span class="n">operation</span><span class="o">.</span><span class="n">status</span> <span class="o">!=</span> <span class="n">Operation</span><span class="o">.</span><span class="n">Status</span><span class="o">.</span><span class="n">DONE</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">operation</span><span class="o">.</span><span class="n">status</span> <span class="o">==</span> <span class="n">Operation</span><span class="o">.</span><span class="n">Status</span><span class="o">.</span><span class="n">RUNNING</span> <span class="ow">or</span> <span class="n">operation</span><span class="o">.</span><span class="n">status</span> <span class="o">==</span> \
+ <span class="n">Operation</span><span class="o">.</span><span class="n">Status</span><span class="o">.</span><span class="n">PENDING</span><span class="p">:</span>
+ <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="n">OPERATIONAL_POLL_INTERVAL</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">exceptions</span><span class="o">.</span><span class="n">GoogleCloudError</span><span class="p">(</span>
+ <span class="s2">"Operation has failed with status: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">operation</span><span class="o">.</span><span class="n">status</span><span class="p">)</span>
+ <span class="c1"># To update status of operation</span>
+ <span class="n">operation</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_operation</span><span class="p">(</span><span class="n">operation</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">operation</span></div>
+
+<div class="viewcode-block" id="GKEClusterHook.get_operation"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_container_hook.GKEClusterHook.get_operation">[docs]</a> <span class="k">def</span> <span class="nf">get_operation</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Fetches the operation from Google Cloud</span>
+<span class="sd"> :param operation_name: Name of operation to fetch</span>
+<span class="sd"> :type operation_name: str</span>
+<span class="sd"> :return: The new, updated operation from Google Cloud</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">get_operation</span><span class="p">(</span><span class="n">project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">zone</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">,</span>
+ <span class="n">operation_id</span><span class="o">=</span><span class="n">operation_name</span><span class="p">)</span></div>
+
+ <span class="k">def</span> <span class="nf">_append_label</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cluster_proto</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">val</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Append labels to provided Cluster Protobuf</span>
+
+<span class="sd"> Labels must fit the regex [a-z]([-a-z0-9]*[a-z0-9])? (current airflow version</span>
+<span class="sd"> string follows semantic versioning spec: x.y.z).</span>
+<span class="sd"> :param cluster_proto: The proto to append resource_label airflow version to</span>
+<span class="sd"> :type cluster_proto: google.cloud.container_v1.types.Cluster</span>
+<span class="sd"> :param key: The key label</span>
+<span class="sd"> :type key: str</span>
+<span class="sd"> :param val:</span>
+<span class="sd"> :type val: str</span>
+<span class="sd"> :return: The cluster proto updated with new label</span>
+<span class="sd"> """</span>
+ <span class="n">val</span> <span class="o">=</span> <span class="n">val</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'.'</span><span class="p">,</span> <span class="s1">'-'</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'+'</span><span class="p">,</span> <span class="s1">'-'</span><span class="p">)</span>
+ <span class="n">cluster_proto</span><span class="o">.</span><span class="n">resource_labels</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="n">key</span><span class="p">:</span> <span class="n">val</span><span class="p">})</span>
+ <span class="k">return</span> <span class="n">cluster_proto</span>
+
+<div class="viewcode-block" id="GKEClusterHook.delete_cluster"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_container_hook.GKEClusterHook.delete_cluster">[docs]</a> <span class="k">def</span> <span class="nf">delete_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">retry</span><span class="o">=</span><span class="n">DEFAULT</span><span c [...]
+ <span class="sd">"""</span>
+<span class="sd"> Deletes the cluster, including the Kubernetes endpoint and all</span>
+<span class="sd"> worker nodes. Firewalls and routes that were configured during</span>
+<span class="sd"> cluster creation are also deleted. Other Google Compute Engine</span>
+<span class="sd"> resources that might be in use by the cluster (e.g. load balancer</span>
+<span class="sd"> resources) will not be deleted if they weren’t present at the</span>
+<span class="sd"> initial create time.</span>
+
+<span class="sd"> :param name: The name of the cluster to delete</span>
+<span class="sd"> :type name: str</span>
+<span class="sd"> :param retry: Retry object used to determine when/if to retry requests.</span>
+<span class="sd"> If None is specified, requests will not be retried.</span>
+<span class="sd"> :type retry: google.api_core.retry.Retry</span>
+<span class="sd"> :param timeout: The amount of time, in seconds, to wait for the request to</span>
+<span class="sd"> complete. Note that if retry is specified, the timeout applies to each</span>
+<span class="sd"> individual attempt.</span>
+<span class="sd"> :type timeout: float</span>
+<span class="sd"> :return: The full url to the delete operation if successful, else None</span>
+<span class="sd"> """</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Deleting (project_id=</span><span class="si">{}</span><span class="s2">, zone=</span><span class="si">{}</span><span class="s2">, cluster_id=</span><span class="si">{}</span><span class="s2">)"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">,</span> <span class="n">name</span><span class="p">))</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">op</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">delete_cluster</span><span class="p">(</span><span class="n">project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">zone</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">,</span>
+ <span class="n">cluster_id</span><span class="o">=</span><span class="n">name</span><span class="p">,</span>
+ <span class="n">retry</span><span class="o">=</span><span class="n">retry</span><span class="p">,</span>
+ <span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span>
+ <span class="n">op</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">wait_for_operation</span><span class="p">(</span><span class="n">op</span><span class="p">)</span>
+ <span class="c1"># Returns server-defined url for the resource</span>
+ <span class="k">return</span> <span class="n">op</span><span class="o">.</span><span class="n">self_link</span>
+ <span class="k">except</span> <span class="n">exceptions</span><span class="o">.</span><span class="n">NotFound</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Assuming Success: '</span> <span class="o">+</span> <span class="n">error</span><span class="o">.</span><span class="n">message</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="GKEClusterHook.create_cluster"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_container_hook.GKEClusterHook.create_cluster">[docs]</a> <span class="k">def</span> <span class="nf">create_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cluster</span><span class="p">,</span> <span class="n">retry</span><span class="o">=</span><span class="n">DEFAULT</span><spa [...]
+ <span class="sd">"""</span>
+<span class="sd"> Creates a cluster, consisting of the specified number and type of Google Compute</span>
+<span class="sd"> Engine instances.</span>
+
+<span class="sd"> :param cluster: A Cluster protobuf or dict. If dict is provided, it must be of</span>
+<span class="sd"> the same form as the protobuf message google.cloud.container_v1.types.Cluster</span>
+<span class="sd"> :type cluster: dict or google.cloud.container_v1.types.Cluster</span>
+<span class="sd"> :param retry: A retry object (google.api_core.retry.Retry) used to retry requests.</span>
+<span class="sd"> If None is specified, requests will not be retried.</span>
+<span class="sd"> :type retry: google.api_core.retry.Retry</span>
+<span class="sd"> :param timeout: The amount of time, in seconds, to wait for the request to</span>
+<span class="sd"> complete. Note that if retry is specified, the timeout applies to each</span>
+<span class="sd"> individual attempt.</span>
+<span class="sd"> :type timeout: float</span>
+<span class="sd"> :return: The full url to the new, or existing, cluster</span>
+<span class="sd"> :raises</span>
+<span class="sd"> ParseError: On JSON parsing problems when trying to convert dict</span>
+<span class="sd"> AirflowException: cluster is not dict type nor Cluster proto type</span>
+<span class="sd"> """</span>
+
+ <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">cluster</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
+ <span class="n">cluster_proto</span> <span class="o">=</span> <span class="n">Cluster</span><span class="p">()</span>
+ <span class="n">cluster</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_dict_to_proto</span><span class="p">(</span><span class="n">py_dict</span><span class="o">=</span><span class="n">cluster</span><span class="p">,</span> <span class="n">proto</span><span class="o">=</span><span class="n">cluster_proto</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">cluster</span><span class="p">,</span> <span class="n">Cluster</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s2">"cluster is not instance of Cluster proto or python dict"</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">_append_label</span><span class="p">(</span><span class="n">cluster</span><span class="p">,</span> <span class="s1">'airflow-version'</span><span class="p">,</span> <span class="s1">'v'</span> <span class="o">+</span> <span class="n">version</span><span class="o">.</span><span class="n">version</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Creating (project_id=</span><span class="si">{}</span><span class="s2">, zone=</span><span class="si">{}</span><span class="s2">, cluster_name=</span><span class="si">{}</span><span class="s2">)"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">,</span>
+ <span class="n">cluster</span><span class="o">.</span><span class="n">name</span><span class="p">))</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">op</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">create_cluster</span><span class="p">(</span><span class="n">project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">zone</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">,</span>
+ <span class="n">cluster</span><span class="o">=</span><span class="n">cluster</span><span class="p">,</span>
+ <span class="n">retry</span><span class="o">=</span><span class="n">retry</span><span class="p">,</span>
+ <span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span>
+ <span class="n">op</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">wait_for_operation</span><span class="p">(</span><span class="n">op</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">op</span><span class="o">.</span><span class="n">target_link</span>
+ <span class="k">except</span> <span class="n">AlreadyExists</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Assuming Success: '</span> <span class="o">+</span> <span class="n">error</span><span class="o">.</span><span class="n">message</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_cluster</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="n">cluster</span><span class="o">.</span><span class="n">name</span><span class="p">)</span><span class="o">.</span><span class="n">self_link</span></div>
+
+<div class="viewcode-block" id="GKEClusterHook.get_cluster"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_container_hook.GKEClusterHook.get_cluster">[docs]</a> <span class="k">def</span> <span class="nf">get_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">retry</span><span class="o">=</span><span class="n">DEFAULT</span><span class="p"> [...]
+ <span class="sd">"""</span>
+<span class="sd"> Gets details of specified cluster</span>
+<span class="sd"> :param name: The name of the cluster to retrieve</span>
+<span class="sd"> :type name: str</span>
+<span class="sd"> :param retry: A retry object used to retry requests. If None is specified,</span>
+<span class="sd"> requests will not be retried.</span>
+<span class="sd"> :type retry: google.api_core.retry.Retry</span>
+<span class="sd"> :param timeout: The amount of time, in seconds, to wait for the request to</span>
+<span class="sd"> complete. Note that if retry is specified, the timeout applies to each</span>
+<span class="sd"> individual attempt.</span>
+<span class="sd"> :type timeout: float</span>
+<span class="sd"> :return: A google.cloud.container_v1.types.Cluster instance</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Fetching cluster (project_id=</span><span class="si">{}</span><span class="s2">, zone=</span><span class="si">{}</span><span class="s2">, cluster_name=</span><span class="si">{}</span><span class="s2">)"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">,</span>
+ <span class="n">name</span><span class="p">))</span>
+
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">get_cluster</span><span class="p">(</span><span class="n">project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">zone</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">,</span>
+ <span class="n">cluster_id</span><span class="o">=</span><span class="n">name</span><span class="p">,</span>
+ <span class="n">retry</span><span class="o">=</span><span class="n">retry</span><span class="p">,</span>
+ <span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span><span class="o">.</span><span class="n">self_link</span></div></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ LANGUAGE:'None',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/gcp_dataflow_hook.html b/_modules/airflow/contrib/hooks/gcp_dataflow_hook.html
index ce6f6ff..808d22f 100644
--- a/_modules/airflow/contrib/hooks/gcp_dataflow_hook.html
+++ b/_modules/airflow/contrib/hooks/gcp_dataflow_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -340,7 +330,8 @@
<span class="sd"> Returns a Google Cloud Storage service object.</span>
<span class="sd"> """</span>
<span class="n">http_authorized</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
- <span class="k">return</span> <span class="n">build</span><span class="p">(</span><span class="s1">'dataflow'</span><span class="p">,</span> <span class="s1">'v1b3'</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">)</span></div>
+ <span class="k">return</span> <span class="n">build</span><span class="p">(</span>
+ <span class="s1">'dataflow'</span><span class="p">,</span> <span class="s1">'v1b3'</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">,</span> <span class="n">cache_discovery</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
<span class="k">def</span> <span class="nf">_start_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span>
<span class="n">command_prefix</span><span class="p">,</span> <span class="n">label_formatter</span><span class="p">):</span>
@@ -434,9 +425,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -448,7 +437,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -467,6 +456,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -479,19 +469,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/gcp_dataproc_hook.html b/_modules/airflow/contrib/hooks/gcp_dataproc_hook.html
index 4ca7edd..d22dfa5 100644
--- a/_modules/airflow/contrib/hooks/gcp_dataproc_hook.html
+++ b/_modules/airflow/contrib/hooks/gcp_dataproc_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -192,6 +182,7 @@
<span class="kn">import</span> <span class="nn">uuid</span>
<span class="kn">from</span> <span class="nn">apiclient.discovery</span> <span class="k">import</span> <span class="n">build</span>
+<span class="kn">from</span> <span class="nn">zope.deprecation</span> <span class="k">import</span> <span class="n">deprecation</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="k">import</span> <span class="n">GoogleCloudBaseHook</span>
<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
@@ -375,7 +366,9 @@
<div class="viewcode-block" id="DataProcHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.gcp_dataproc_hook.DataProcHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""Returns a Google Cloud Dataproc service object."""</span>
<span class="n">http_authorized</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
- <span class="k">return</span> <span class="n">build</span><span class="p">(</span><span class="s1">'dataproc'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">api_version</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">)</span></div>
+ <span class="k">return</span> <span class="n">build</span><span class="p">(</span>
+ <span class="s1">'dataproc'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">api_version</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">,</span>
+ <span class="n">cache_discovery</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
<span class="k">def</span> <span class="nf">get_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">region</span><span class="p">,</span> <span class="n">cluster_name</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">regions</span><span class="p">()</span><span class="o">.</span><span class="n">clusters</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
@@ -393,16 +386,23 @@
<span class="k">return</span> <span class="n">_DataProcJobBuilder</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">cluster_name</span><span class="p">,</span>
<span class="n">job_type</span><span class="p">,</span> <span class="n">properties</span><span class="p">)</span>
-<div class="viewcode-block" id="DataProcHook.await"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.gcp_dataproc_hook.DataProcHook.await">[docs]</a> <span class="k">def</span> <span class="nf">await</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">operation</span><span class="p">):</span>
+<div class="viewcode-block" id="DataProcHook.wait"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.gcp_dataproc_hook.DataProcHook.wait">[docs]</a> <span class="k">def</span> <span class="nf">wait</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">operation</span><span class="p">):</span>
<span class="sd">"""Awaits for Google Cloud Dataproc Operation to complete."""</span>
<span class="n">submitted</span> <span class="o">=</span> <span class="n">_DataProcOperation</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">(),</span> <span class="n">operation</span><span class="p">)</span>
<span class="n">submitted</span><span class="o">.</span><span class="n">wait_for_done</span><span class="p">()</span></div></div>
+
+
+<span class="nb">setattr</span><span class="p">(</span>
+ <span class="n">DataProcHook</span><span class="p">,</span>
+ <span class="s2">"await"</span><span class="p">,</span>
+ <span class="n">deprecation</span><span class="o">.</span><span class="n">deprecated</span><span class="p">(</span>
+ <span class="n">DataProcHook</span><span class="o">.</span><span class="n">wait</span><span class="p">,</span> <span class="s2">"renamed to 'wait' for Python3.7 compatability"</span>
+ <span class="p">),</span>
+<span class="p">)</span>
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -414,7 +414,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -433,6 +433,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -445,19 +446,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/gcp_mlengine_hook.html b/_modules/airflow/contrib/hooks/gcp_mlengine_hook.html
index df9ea25..b961738 100644
--- a/_modules/airflow/contrib/hooks/gcp_mlengine_hook.html
+++ b/_modules/airflow/contrib/hooks/gcp_mlengine_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -188,7 +178,6 @@
<span class="kn">import</span> <span class="nn">time</span>
<span class="kn">from</span> <span class="nn">apiclient</span> <span class="k">import</span> <span class="n">errors</span>
<span class="kn">from</span> <span class="nn">apiclient.discovery</span> <span class="k">import</span> <span class="n">build</span>
-<span class="kn">from</span> <span class="nn">oauth2client.client</span> <span class="k">import</span> <span class="n">GoogleCredentials</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="k">import</span> <span class="n">GoogleCloudBaseHook</span>
<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
@@ -226,8 +215,8 @@
<span class="sd">"""</span>
<span class="sd"> Returns a Google MLEngine service object.</span>
<span class="sd"> """</span>
- <span class="n">credentials</span> <span class="o">=</span> <span class="n">GoogleCredentials</span><span class="o">.</span><span class="n">get_application_default</span><span class="p">()</span>
- <span class="k">return</span> <span class="n">build</span><span class="p">(</span><span class="s1">'ml'</span><span class="p">,</span> <span class="s1">'v1'</span><span class="p">,</span> <span class="n">credentials</span><span class="o">=</span><span class="n">credentials</span><span class="p">)</span></div>
+ <span class="n">authed_http</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">build</span><span class="p">(</span><span class="s1">'ml'</span><span class="p">,</span> <span class="s1">'v1'</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">authed_http</span><span class="p">,</span> <span class="n">cache_discovery</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
<div class="viewcode-block" id="MLEngineHook.create_job"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_mlengine_hook.MLEngineHook.create_job">[docs]</a> <span class="k">def</span> <span class="nf">create_job</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="p">,</span> <span class="n">use_existing_job_fn</span><span c [...]
<span class="sd">"""</span>
@@ -439,9 +428,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -453,7 +440,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -472,6 +459,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -484,19 +472,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/gcp_pubsub_hook.html b/_modules/airflow/contrib/hooks/gcp_pubsub_hook.html
index 19713d1..1cb0cac 100644
--- a/_modules/airflow/contrib/hooks/gcp_pubsub_hook.html
+++ b/_modules/airflow/contrib/hooks/gcp_pubsub_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -224,7 +214,8 @@
<span class="sd"> :rtype: apiclient.discovery.Resource</span>
<span class="sd"> """</span>
<span class="n">http_authorized</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
- <span class="k">return</span> <span class="n">build</span><span class="p">(</span><span class="s1">'pubsub'</span><span class="p">,</span> <span class="s1">'v1'</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">)</span></div>
+ <span class="k">return</span> <span class="n">build</span><span class="p">(</span>
+ <span class="s1">'pubsub'</span><span class="p">,</span> <span class="s1">'v1'</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">,</span> <span class="n">cache_discovery</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
<div class="viewcode-block" id="PubSubHook.publish"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.gcp_pubsub_hook.PubSubHook.publish">[docs]</a> <span class="k">def</span> <span class="nf">publish</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project</span><span class="p">,</span> <span class="n">topic</span><span class="p">,</span> <span class="n">messages</span><span class="p">):</span>
<span class="sd">"""Publishes messages to a Pub/Sub topic.</span>
@@ -456,9 +447,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -470,7 +459,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -489,6 +478,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -501,19 +491,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/gcs_hook.html b/_modules/airflow/contrib/hooks/gcs_hook.html
index 5db94f8..c4896fb 100644
--- a/_modules/airflow/contrib/hooks/gcs_hook.html
+++ b/_modules/airflow/contrib/hooks/gcs_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -215,7 +205,8 @@
<span class="sd"> Returns a Google Cloud Storage service object.</span>
<span class="sd"> """</span>
<span class="n">http_authorized</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
- <span class="k">return</span> <span class="n">build</span><span class="p">(</span><span class="s1">'storage'</span><span class="p">,</span> <span class="s1">'v1'</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">)</span></div>
+ <span class="k">return</span> <span class="n">build</span><span class="p">(</span>
+ <span class="s1">'storage'</span><span class="p">,</span> <span class="s1">'v1'</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">,</span> <span class="n">cache_discovery</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
<span class="c1"># pylint:disable=redefined-builtin</span>
<div class="viewcode-block" id="GoogleCloudStorageHook.copy"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.copy">[docs]</a> <span class="k">def</span> <span class="nf">copy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">source_bucket</span><span class="p">,</span> <span class="n">source_object</span><span class="p">,</span> <span class="n">destination_bucket</span><s [...]
@@ -238,8 +229,9 @@
<span class="sd"> """</span>
<span class="n">destination_bucket</span> <span class="o">=</span> <span class="n">destination_bucket</span> <span class="ow">or</span> <span class="n">source_bucket</span>
<span class="n">destination_object</span> <span class="o">=</span> <span class="n">destination_object</span> <span class="ow">or</span> <span class="n">source_object</span>
- <span class="k">if</span> <span class="p">(</span><span class="n">source_bucket</span> <span class="o">==</span> <span class="n">destination_bucket</span> <span class="ow">and</span>
- <span class="n">source_object</span> <span class="o">==</span> <span class="n">destination_object</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">source_bucket</span> <span class="o">==</span> <span class="n">destination_bucket</span> <span class="ow">and</span> \
+ <span class="n">source_object</span> <span class="o">==</span> <span class="n">destination_object</span><span class="p">:</span>
+
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
<span class="s1">'Either source/destination bucket or source/destination object '</span>
<span class="s1">'must be different, not both the same: bucket=</span><span class="si">%s</span><span class="s1">, object=</span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span>
@@ -355,10 +347,16 @@
<span class="sd"> """</span>
<span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
<span class="n">media</span> <span class="o">=</span> <span class="n">MediaFileUpload</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="n">mime_type</span><span class="p">)</span>
- <span class="n">response</span> <span class="o">=</span> <span class="n">service</span> \
- <span class="o">.</span><span class="n">objects</span><span class="p">()</span> \
- <span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="n">bucket</span><span class="o">=</span><span class="n">bucket</span><span class="p">,</span> <span class="n">name</span><span class="o">=</span><span class="nb">object</span><span class="p">,</span> <span class="n">media_body</span><span class="o">=</span><span class="n">media</span><span class="p">)</span> \
- <span class="o">.</span><span class="n">execute</span><span class="p">()</span></div>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">service</span> \
+ <span class="o">.</span><span class="n">objects</span><span class="p">()</span> \
+ <span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="n">bucket</span><span class="o">=</span><span class="n">bucket</span><span class="p">,</span> <span class="n">name</span><span class="o">=</span><span class="nb">object</span><span class="p">,</span> <span class="n">media_body</span><span class="o">=</span><span class="n">media</span><span class="p">)</span> \
+ <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">return</span> <span class="kc">True</span>
+ <span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">ex</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">ex</span><span class="o">.</span><span class="n">resp</span><span class="p">[</span><span class="s1">'status'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'404'</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">False</span>
+ <span class="k">raise</span></div>
<span class="c1"># pylint:disable=redefined-builtin</span>
<div class="viewcode-block" id="GoogleCloudStorageHook.exists"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.exists">[docs]</a> <span class="k">def</span> <span class="nf">exists</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">):</span>
@@ -458,7 +456,8 @@
<span class="sd"> :type versions: boolean</span>
<span class="sd"> :param maxResults: max count of items to return in a single page of responses</span>
<span class="sd"> :type maxResults: integer</span>
-<span class="sd"> :param prefix: prefix string which filters objects whose name begin with this prefix</span>
+<span class="sd"> :param prefix: prefix string which filters objects whose name begin with</span>
+<span class="sd"> this prefix</span>
<span class="sd"> :type prefix: string</span>
<span class="sd"> :param delimiter: filters objects based on the delimiter (for e.g '.csv')</span>
<span class="sd"> :type delimiter: string</span>
@@ -510,7 +509,9 @@
<span class="sd"> :type object: string</span>
<span class="sd"> """</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Checking the file size of object: </span><span class="si">%s</span><span class="s1"> in bucket: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">bucket</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Checking the file size of object: </span><span class="si">%s</span><span class="s1"> in bucket: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="nb">object</span><span class="p">,</span>
+ <span class="n">bucket</span><span class="p">)</span>
<span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">response</span> <span class="o">=</span> <span class="n">service</span><span class="o">.</span><span class="n">objects</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
@@ -698,9 +699,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -712,7 +711,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -731,6 +730,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -743,19 +743,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/jenkins_hook.html b/_modules/airflow/contrib/hooks/jenkins_hook.html
index 783bfa6..1ffe07f 100644
--- a/_modules/airflow/contrib/hooks/jenkins_hook.html
+++ b/_modules/airflow/contrib/hooks/jenkins_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -220,9 +210,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -234,7 +222,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -253,6 +241,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -265,19 +254,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/jira_hook.html b/_modules/airflow/contrib/hooks/jira_hook.html
index 7dc6dc0..cdf8dc2 100644
--- a/_modules/airflow/contrib/hooks/jira_hook.html
+++ b/_modules/airflow/contrib/hooks/jira_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -203,9 +193,11 @@
<span class="sd"> :type jira_conn_id: string</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
- <span class="n">jira_conn_id</span><span class="o">=</span><span class="s1">'jira_default'</span><span class="p">):</span>
+ <span class="n">jira_conn_id</span><span class="o">=</span><span class="s1">'jira_default'</span><span class="p">,</span>
+ <span class="n">proxies</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">JiraHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">jira_conn_id</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">jira_conn_id</span> <span class="o">=</span> <span class="n">jira_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">proxies</span> <span class="o">=</span> <span class="n">proxies</span>
<span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="kc">None</span>
<span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
@@ -244,7 +236,8 @@
<span class="n">options</span><span class="o">=</span><span class="n">extra_options</span><span class="p">,</span>
<span class="n">basic_auth</span><span class="o">=</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">),</span>
<span class="n">get_server_info</span><span class="o">=</span><span class="n">get_server_info</span><span class="p">,</span>
- <span class="n">validate</span><span class="o">=</span><span class="n">validate</span><span class="p">)</span>
+ <span class="n">validate</span><span class="o">=</span><span class="n">validate</span><span class="p">,</span>
+ <span class="n">proxies</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">proxies</span><span class="p">)</span>
<span class="k">except</span> <span class="n">JIRAError</span> <span class="k">as</span> <span class="n">jira_error</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Failed to create jira client, jira error: </span><span class="si">%s</span><span class="s1">'</span>
<span class="o">%</span> <span class="nb">str</span><span class="p">(</span><span class="n">jira_error</span><span class="p">))</span>
@@ -256,9 +249,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -270,7 +261,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -289,6 +280,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -301,19 +293,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/mongo_hook.html b/_modules/airflow/contrib/hooks/mongo_hook.html
new file mode 100644
index 0000000..e1c22e1
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/mongo_hook.html
@@ -0,0 +1,337 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.mongo_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.mongo_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.mongo_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="kn">from</span> <span class="nn">ssl</span> <span class="k">import</span> <span class="n">CERT_NONE</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">pymongo</span> <span class="k">import</span> <span class="n">MongoClient</span>
+
+
+<div class="viewcode-block" id="MongoHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.mongo_hook.MongoHook">[docs]</a><span class="k">class</span> <span class="nc">MongoHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> PyMongo Wrapper to Interact With Mongo Database</span>
+<span class="sd"> Mongo Connection Documentation</span>
+<span class="sd"> https://docs.mongodb.com/manual/reference/connection-string/index.html</span>
+<span class="sd"> You can specify connection string options in extra field of your connection</span>
+<span class="sd"> https://docs.mongodb.com/manual/reference/connection-string/index.html#connection-string-options</span>
+<span class="sd"> ex.</span>
+<span class="sd"> {replicaSet: test, ssl: True, connectTimeoutMS: 30000}</span>
+<span class="sd"> """</span>
+ <span class="n">conn_type</span> <span class="o">=</span> <span class="s1">'MongoDb'</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn_id</span><span class="o">=</span><span class="s1">'mongo_default'</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">MongoHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">source</span><span class="o">=</span><span class="s1">'mongo'</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">mongo_conn_id</span> <span class="o">=</span> <span class="n">conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">connection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">conn_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">extras</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">extra_dejson</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="kc">None</span>
+
+<div class="viewcode-block" id="MongoHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.mongo_hook.MongoHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Fetches PyMongo Client</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span>
+
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span>
+
+ <span class="n">uri</span> <span class="o">=</span> <span class="s1">'mongodb://</span><span class="si">{creds}{host}{port}</span><span class="s1">/</span><span class="si">{database}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">creds</span><span class="o">=</span><span class="s1">'</span><span class="si">{}</span><span class="s1">:</span><span class="si">{}</span><span class="s1">@'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span>
+ <span class="p">)</span> <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="s1">''</span><span class="p">,</span>
+
+ <span class="n">host</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">,</span>
+ <span class="n">port</span><span class="o">=</span><span class="s1">''</span> <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="s1">':</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">conn</span><span class="o"> [...]
+ <span class="n">database</span><span class="o">=</span><span class="s1">''</span> <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">schema</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">conn</span><span class="o">.</span><span class="n">schema</span>
+ <span class="p">)</span>
+
+ <span class="c1"># Mongo Connection Options dict that is unpacked when passed to MongoClient</span>
+ <span class="n">options</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">extras</span>
+
+ <span class="c1"># If we are using SSL disable requiring certs from specific hostname</span>
+ <span class="k">if</span> <span class="n">options</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'ssl'</span><span class="p">,</span> <span class="kc">False</span><span class="p">):</span>
+ <span class="n">options</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s1">'ssl_cert_reqs'</span><span class="p">:</span> <span class="n">CERT_NONE</span><span class="p">})</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="n">MongoClient</span><span class="p">(</span><span class="n">uri</span><span class="p">,</span> <span class="o">**</span><span class="n">options</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span></div>
+
+<div class="viewcode-block" id="MongoHook.get_collection"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.mongo_hook.MongoHook.get_collection">[docs]</a> <span class="k">def</span> <span class="nf">get_collection</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">mongo_collection</span><span class="p">,</span> <span class="n">mongo_db</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Fetches a mongo collection object for querying.</span>
+
+<span class="sd"> Uses connection schema as DB unless specified.</span>
+<span class="sd"> """</span>
+ <span class="n">mongo_db</span> <span class="o">=</span> <span class="n">mongo_db</span> <span class="k">if</span> <span class="n">mongo_db</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">schema</span>
+ <span class="n">mongo_conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+ <span class="k">return</span> <span class="n">mongo_conn</span><span class="o">.</span><span class="n">get_database</span><span class="p">(</span><span class="n">mongo_db</span><span class="p">)</span><span class="o">.</span><span class="n">get_collection</span><span class="p">(</span><span class="n">mongo_collection</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="MongoHook.aggregate"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.mongo_hook.MongoHook.aggregate">[docs]</a> <span class="k">def</span> <span class="nf">aggregate</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">mongo_collection</span><span class="p">,</span> <span class="n">aggregate_query</span><span class="p">,</span> <span class="n">mongo_db</span><span class="o">=</span [...]
+ <span class="sd">"""</span>
+<span class="sd"> Runs an aggregation pipeline and returns the results</span>
+<span class="sd"> https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.aggregate</span>
+<span class="sd"> https://api.mongodb.com/python/current/examples/aggregation.html</span>
+<span class="sd"> """</span>
+ <span class="n">collection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_collection</span><span class="p">(</span><span class="n">mongo_collection</span><span class="p">,</span> <span class="n">mongo_db</span><span class="o">=</span><span class="n">mongo_db</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">collection</span><span class="o">.</span><span class="n">aggregate</span><span class="p">(</span><span class="n">aggregate_query</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="MongoHook.find"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.mongo_hook.MongoHook.find">[docs]</a> <span class="k">def</span> <span class="nf">find</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">mongo_collection</span><span class="p">,</span> <span class="n">query</span><span class="p">,</span> <span class="n">find_one</span><span class="o">=</span><span class="kc">False</ [...]
+ <span class="sd">"""</span>
+<span class="sd"> Runs a mongo find query and returns the results</span>
+<span class="sd"> https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.find</span>
+<span class="sd"> """</span>
+ <span class="n">collection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_collection</span><span class="p">(</span><span class="n">mongo_collection</span><span class="p">,</span> <span class="n">mongo_db</span><span class="o">=</span><span class="n">mongo_db</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">find_one</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">collection</span><span class="o">.</span><span class="n">find_one</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">collection</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="MongoHook.insert_one"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.mongo_hook.MongoHook.insert_one">[docs]</a> <span class="k">def</span> <span class="nf">insert_one</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">mongo_collection</span><span class="p">,</span> <span class="n">doc</span><span class="p">,</span> <span class="n">mongo_db</span><span class="o">=</span><span cl [...]
+ <span class="sd">"""</span>
+<span class="sd"> Inserts a single document into a mongo collection</span>
+<span class="sd"> https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.insert_one</span>
+<span class="sd"> """</span>
+ <span class="n">collection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_collection</span><span class="p">(</span><span class="n">mongo_collection</span><span class="p">,</span> <span class="n">mongo_db</span><span class="o">=</span><span class="n">mongo_db</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">collection</span><span class="o">.</span><span class="n">insert_one</span><span class="p">(</span><span class="n">doc</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="MongoHook.insert_many"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.mongo_hook.MongoHook.insert_many">[docs]</a> <span class="k">def</span> <span class="nf">insert_many</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">mongo_collection</span><span class="p">,</span> <span class="n">docs</span><span class="p">,</span> <span class="n">mongo_db</span><span class="o">=</span><spa [...]
+ <span class="sd">"""</span>
+<span class="sd"> Inserts many docs into a mongo collection.</span>
+<span class="sd"> https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.insert_many</span>
+<span class="sd"> """</span>
+ <span class="n">collection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_collection</span><span class="p">(</span><span class="n">mongo_collection</span><span class="p">,</span> <span class="n">mongo_db</span><span class="o">=</span><span class="n">mongo_db</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">collection</span><span class="o">.</span><span class="n">insert_many</span><span class="p">(</span><span class="n">docs</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ LANGUAGE:'None',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/pinot_hook.html b/_modules/airflow/contrib/hooks/pinot_hook.html
index a34b554..2c03016 100644
--- a/_modules/airflow/contrib/hooks/pinot_hook.html
+++ b/_modules/airflow/contrib/hooks/pinot_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -266,20 +256,18 @@
<span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql</span><span class="p">)</span>
<span class="k">return</span> <span class="n">cur</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span></div>
- <span class="k">def</span> <span class="nf">set_autocommit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn</span><span class="p">,</span> <span class="n">autocommit</span><span class="p">):</span>
- <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">()</span>
-
- <span class="k">def</span> <span class="nf">get_pandas_df</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
- <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">()</span>
+<div class="viewcode-block" id="PinotDbApiHook.set_autocommit"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.pinot_hook.PinotDbApiHook.set_autocommit">[docs]</a> <span class="k">def</span> <span class="nf">set_autocommit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn</span><span class="p">,</span> <span class="n">autocommit</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">()</span></div>
- <span class="k">def</span> <span class="nf">insert_rows</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">rows</span><span class="p">,</span> <span class="n">target_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">commit_every</span><span class="o">=</span><span class="mi">1000</span><span class="p">):</span>
+<div class="viewcode-block" id="PinotDbApiHook.get_pandas_df"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.pinot_hook.PinotDbApiHook.get_pandas_df">[docs]</a> <span class="k">def</span> <span class="nf">get_pandas_df</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="PinotDbApiHook.insert_rows"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.pinot_hook.PinotDbApiHook.insert_rows">[docs]</a> <span class="k">def</span> <span class="nf">insert_rows</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">rows</span><span class="p">,</span> <span class="n">target_fields</span><span class="o">=</span> [...]
+ <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">()</span></div></div>
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -291,7 +279,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -310,6 +298,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -322,19 +311,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/qubole_hook.html b/_modules/airflow/contrib/hooks/qubole_hook.html
index c5fbc73..4968269 100644
--- a/_modules/airflow/contrib/hooks/qubole_hook.html
+++ b/_modules/airflow/contrib/hooks/qubole_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -265,7 +255,7 @@
<span class="n">log</span> <span class="o">=</span> <span class="n">LoggingMixin</span><span class="p">()</span><span class="o">.</span><span class="n">log</span>
<span class="k">if</span> <span class="n">cmd</span><span class="o">.</span><span class="n">status</span> <span class="o">==</span> <span class="s1">'done'</span><span class="p">:</span>
<span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Command ID: </span><span class="si">%s</span><span class="s1"> has been succeeded, hence marking this '</span>
- <span class="s1">'TI as Success.'</span><span class="p">,</span> <span class="n">cmd_id</span><span class="p">)</span>
+ <span class="s1">'TI as Success.'</span><span class="p">,</span> <span class="n">cmd_id</span><span class="p">)</span>
<span class="n">ti</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span>
<span class="k">elif</span> <span class="n">cmd</span><span class="o">.</span><span class="n">status</span> <span class="o">==</span> <span class="s1">'running'</span><span class="p">:</span>
<span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Cancelling the Qubole Command Id: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">cmd_id</span><span class="p">)</span>
@@ -359,10 +349,10 @@
<span class="n">inplace_args</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">tags</span> <span class="o">=</span> <span class="nb">set</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="n">context</span><span class="p">[</span><span class="s1">'run_id'</span><span class="p">]])</span>
- <span class="k">for</span> <span class="n">k</span><span class="p">,</span><span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+ <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="k">if</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">COMMAND_ARGS</span><span class="p">[</span><span class="n">cmd_type</span><span class="p">]:</span>
<span class="k">if</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">HYPHEN_ARGS</span><span class="p">:</span>
- <span class="n">args</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"--</span><span class="si">{0}</span><span class="s2">=</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">k</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'_'</span><span class=" [...]
+ <span class="n">args</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"--</span><span class="si">{0}</span><span class="s2">=</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">k</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'_'</span><span class=" [...]
<span class="k">elif</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">POSITIONAL_ARGS</span><span class="p">:</span>
<span class="n">inplace_args</span> <span class="o">=</span> <span class="n">v</span>
<span class="k">elif</span> <span class="n">k</span> <span class="o">==</span> <span class="s1">'tags'</span><span class="p">:</span>
@@ -372,12 +362,12 @@
<span class="k">for</span> <span class="n">val</span> <span class="ow">in</span> <span class="n">v</span><span class="p">:</span>
<span class="n">tags</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">val</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
- <span class="n">args</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"--</span><span class="si">{0}</span><span class="s2">=</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">k</span><span class="p">,</span><span class="n">v</span><span class="p">))</span>
+ <span class="n">args</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"--</span><span class="si">{0}</span><span class="s2">=</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="n">v</span><span class="p">))</span>
<span class="k">if</span> <span class="n">k</span> <span class="o">==</span> <span class="s1">'notify'</span> <span class="ow">and</span> <span class="n">v</span> <span class="ow">is</span> <span class="kc">True</span><span class="p">:</span>
<span class="n">args</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"--notify"</span><span class="p">)</span>
- <span class="n">args</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"--tags=</span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="s1">','</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="kc">None</span><span class="p">, [...]
+ <span class="n">args</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"--tags=</span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="s1">','</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="kc">None</span><span class="p">, [...]
<span class="k">if</span> <span class="n">inplace_args</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">args</span> <span class="o">+=</span> <span class="n">inplace_args</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">' '</span><span class="p">)</span>
@@ -386,9 +376,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -400,7 +388,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -419,6 +407,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -431,19 +420,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/redis_hook.html b/_modules/airflow/contrib/hooks/redis_hook.html
index 1d0c202..e559ff0 100644
--- a/_modules/airflow/contrib/hooks/redis_hook.html
+++ b/_modules/airflow/contrib/hooks/redis_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -265,9 +255,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -279,7 +267,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -298,6 +286,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -310,19 +299,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/redshift_hook.html b/_modules/airflow/contrib/hooks/redshift_hook.html
index 314fd9e..e587dc2 100644
--- a/_modules/airflow/contrib/hooks/redshift_hook.html
+++ b/_modules/airflow/contrib/hooks/redshift_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -285,9 +275,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -299,7 +287,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -318,6 +306,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -330,19 +319,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/segment_hook.html b/_modules/airflow/contrib/hooks/segment_hook.html
new file mode 100644
index 0000000..ccb8fd0
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/segment_hook.html
@@ -0,0 +1,310 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.segment_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.segment_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.segment_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+<span class="sd">"""</span>
+<span class="sd">This module contains a Segment Hook</span>
+<span class="sd">which allows you to connect to your Segment account,</span>
+<span class="sd">retrieve data from it or write to that file.</span>
+
+<span class="sd">NOTE: this hook also relies on the Segment analytics package:</span>
+<span class="sd"> https://github.com/segmentio/analytics-python</span>
+<span class="sd">"""</span>
+<span class="kn">import</span> <span class="nn">analytics</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+
+<div class="viewcode-block" id="SegmentHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.segment_hook.SegmentHook">[docs]</a><span class="k">class</span> <span class="nc">SegmentHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">segment_conn_id</span><span class="o">=</span><span class="s1">'segment_default'</span><span class="p">,</span>
+ <span class="n">segment_debug_mode</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span>
+ <span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Create new connection to Segment</span>
+<span class="sd"> and allows you to pull data out of Segment or write to it.</span>
+
+<span class="sd"> You can then use that file with other</span>
+<span class="sd"> Airflow operators to move the data around or interact with segment.</span>
+
+<span class="sd"> :param segment_conn_id: the name of the connection that has the parameters</span>
+<span class="sd"> we need to connect to Segment.</span>
+<span class="sd"> The connection should be type `json` and include a</span>
+<span class="sd"> write_key security token in the `Extras` field.</span>
+<span class="sd"> :type segment_conn_id: str</span>
+<span class="sd"> :param segment_debug_mode: Determines whether Segment should run in debug mode.</span>
+<span class="sd"> Defaults to False</span>
+<span class="sd"> :type segment_debug_mode: boolean</span>
+<span class="sd"> .. note::</span>
+<span class="sd"> You must include a JSON structure in the `Extras` field.</span>
+<span class="sd"> We need a user's security token to connect to Segment.</span>
+<span class="sd"> So we define it in the `Extras` field as:</span>
+<span class="sd"> `{"write_key":"YOUR_SECURITY_TOKEN"}`</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">segment_conn_id</span> <span class="o">=</span> <span class="n">segment_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">segment_debug_mode</span> <span class="o">=</span> <span class="n">segment_debug_mode</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_args</span> <span class="o">=</span> <span class="n">args</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_kwargs</span> <span class="o">=</span> <span class="n">kwargs</span>
+
+ <span class="c1"># get the connection parameters</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">connection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">segment_conn_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">extras</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">extra_dejson</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">write_key</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">extras</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'write_key'</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">write_key</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'No Segment write key provided'</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Setting write key for Segment analytics connection'</span><span class="p">)</span>
+ <span class="n">analytics</span><span class="o">.</span><span class="n">debug</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">segment_debug_mode</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">segment_debug_mode</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Setting Segment analytics connection to debug mode'</span><span class="p">)</span>
+ <span class="n">analytics</span><span class="o">.</span><span class="n">on_error</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">on_error</span>
+ <span class="n">analytics</span><span class="o">.</span><span class="n">write_key</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">write_key</span>
+ <span class="k">return</span> <span class="n">analytics</span>
+
+<div class="viewcode-block" id="SegmentHook.on_error"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.segment_hook.SegmentHook.on_error">[docs]</a> <span class="k">def</span> <span class="nf">on_error</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">error</span><span class="p">,</span> <span class="n">items</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Handles error callbacks when using Segment with segment_debug_mode set to True</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">'Encountered Segment error: </span><span class="si">{segment_error}</span><span class="s1"> with '</span>
+ <span class="s1">'items: </span><span class="si">{with_items}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">segment_error</span><span class="o">=</span><span class="n">error</span><span class="p">,</span>
+ <span class="n">with_items</span><span class="o">=</span><span class="n">items</span><span class="p">))</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Segment error: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="p">))</span></div></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ LANGUAGE:'None',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/sftp_hook.html b/_modules/airflow/contrib/hooks/sftp_hook.html
index 6b048cb..efc5485 100644
--- a/_modules/airflow/contrib/hooks/sftp_hook.html
+++ b/_modules/airflow/contrib/hooks/sftp_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -341,9 +331,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -355,7 +343,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -374,6 +362,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -386,19 +375,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/slack_webhook_hook.html b/_modules/airflow/contrib/hooks/slack_webhook_hook.html
index e3e9dfc..8112c68 100644
--- a/_modules/airflow/contrib/hooks/slack_webhook_hook.html
+++ b/_modules/airflow/contrib/hooks/slack_webhook_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -301,9 +291,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -315,7 +303,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -334,6 +322,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -346,19 +335,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/snowflake_hook.html b/_modules/airflow/contrib/hooks/snowflake_hook.html
new file mode 100644
index 0000000..aa15463
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/snowflake_hook.html
@@ -0,0 +1,316 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.snowflake_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.snowflake_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.snowflake_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">snowflake.connector</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="k">import</span> <span class="n">DbApiHook</span>
+
+
+<div class="viewcode-block" id="SnowflakeHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.snowflake_hook.SnowflakeHook">[docs]</a><span class="k">class</span> <span class="nc">SnowflakeHook</span><span class="p">(</span><span class="n">DbApiHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with Snowflake.</span>
+
+<span class="sd"> get_sqlalchemy_engine() depends on snowflake-sqlalchemy</span>
+
+<span class="sd"> """</span>
+
+ <span class="n">conn_name_attr</span> <span class="o">=</span> <span class="s1">'snowflake_conn_id'</span>
+ <span class="n">default_conn_name</span> <span class="o">=</span> <span class="s1">'snowflake_default'</span>
+ <span class="n">supports_autocommit</span> <span class="o">=</span> <span class="kc">True</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">SnowflakeHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">account</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s2">"account"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">warehouse</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s2">"warehouse"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">database</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s2">"database"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_get_conn_params</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> one method to fetch connection params as a dict</span>
+<span class="sd"> used in get_uri() and get_connection()</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">snowflake_conn_id</span><span class="p">)</span>
+ <span class="n">account</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'account'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="n">warehouse</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'warehouse'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="n">database</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'database'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+
+ <span class="n">conn_config</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s2">"user"</span><span class="p">:</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span>
+ <span class="s2">"password"</span><span class="p">:</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span> <span class="ow">or</span> <span class="s1">''</span><span class="p">,</span>
+ <span class="s2">"schema"</span><span class="p">:</span> <span class="n">conn</span><span class="o">.</span><span class="n">schema</span> <span class="ow">or</span> <span class="s1">''</span><span class="p">,</span>
+ <span class="s2">"database"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">database</span> <span class="ow">or</span> <span class="n">database</span> <span class="ow">or</span> <span class="s1">''</span><span class="p">,</span>
+ <span class="s2">"account"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">account</span> <span class="ow">or</span> <span class="n">account</span> <span class="ow">or</span> <span class="s1">''</span><span class="p">,</span>
+ <span class="s2">"warehouse"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">warehouse</span> <span class="ow">or</span> <span class="n">warehouse</span> <span class="ow">or</span> <span class="s1">''</span>
+ <span class="p">}</span>
+ <span class="k">return</span> <span class="n">conn_config</span>
+
+<div class="viewcode-block" id="SnowflakeHook.get_uri"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.snowflake_hook.SnowflakeHook.get_uri">[docs]</a> <span class="k">def</span> <span class="nf">get_uri</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> override DbApiHook get_uri method for get_sqlalchemy_engine()</span>
+<span class="sd"> """</span>
+ <span class="n">conn_config</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_conn_params</span><span class="p">()</span>
+ <span class="n">uri</span> <span class="o">=</span> <span class="s1">'snowflake://</span><span class="si">{user}</span><span class="s1">:</span><span class="si">{password}</span><span class="s1">@</span><span class="si">{account}</span><span class="s1">/</span><span class="si">{database}</span><span class="s1">/'</span>
+ <span class="n">uri</span> <span class="o">+=</span> <span class="s1">'</span><span class="si">{schema}</span><span class="s1">?warehouse=</span><span class="si">{warehouse}</span><span class="s1">'</span>
+ <span class="k">return</span> <span class="n">uri</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="o">**</span><span class="n">conn_config</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="SnowflakeHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.snowflake_hook.SnowflakeHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns a snowflake.connection object</span>
+<span class="sd"> """</span>
+ <span class="n">conn_config</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_conn_params</span><span class="p">()</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="n">snowflake</span><span class="o">.</span><span class="n">connector</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="o">**</span><span class="n">conn_config</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">conn</span></div>
+
+ <span class="k">def</span> <span class="nf">_get_aws_credentials</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> returns aws_access_key_id, aws_secret_access_key</span>
+<span class="sd"> from extra</span>
+
+<span class="sd"> intended to be used by external import and export statements</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">snowflake_conn_id</span><span class="p">:</span>
+ <span class="n">connection_object</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">snowflake_conn_id</span><span class="p">)</span>
+ <span class="k">if</span> <span class="s1">'aws_secret_access_key'</span> <span class="ow">in</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">:</span>
+ <span class="n">aws_access_key_id</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="s1">'aws_access_key_id'</span><span class="p">)</span>
+ <span class="n">aws_secret_access_key</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="s1">'aws_secret_access_key'</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">aws_access_key_id</span><span class="p">,</span> <span class="n">aws_secret_access_key</span>
+
+<div class="viewcode-block" id="SnowflakeHook.set_autocommit"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.snowflake_hook.SnowflakeHook.set_autocommit">[docs]</a> <span class="k">def</span> <span class="nf">set_autocommit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn</span><span class="p">,</span> <span class="n">autocommit</span><span class="p">):</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">autocommit</span><span class="p">(</span><span class="n">autocommit</span><span class="p">)</span></div></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ LANGUAGE:'None',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/spark_jdbc_hook.html b/_modules/airflow/contrib/hooks/spark_jdbc_hook.html
index c22b1e5..f6fe261 100644
--- a/_modules/airflow/contrib/hooks/spark_jdbc_hook.html
+++ b/_modules/airflow/contrib/hooks/spark_jdbc_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -418,9 +408,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -432,7 +420,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -451,6 +439,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -463,19 +452,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/spark_sql_hook.html b/_modules/airflow/contrib/hooks/spark_sql_hook.html
index 08d3147..ba78c7b 100644
--- a/_modules/airflow/contrib/hooks/spark_sql_hook.html
+++ b/_modules/airflow/contrib/hooks/spark_sql_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -192,7 +182,6 @@
<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
<div class="viewcode-block" id="SparkSqlHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.spark_sql_hook.SparkSqlHook">[docs]</a><span class="k">class</span> <span class="nc">SparkSqlHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
@@ -205,9 +194,11 @@
<span class="sd"> :type conf: str (format: PROP=VALUE)</span>
<span class="sd"> :param conn_id: connection_id string</span>
<span class="sd"> :type conn_id: str</span>
-<span class="sd"> :param total_executor_cores: (Standalone & Mesos only) Total cores for all executors (Default: all the available cores on the worker)</span>
+<span class="sd"> :param total_executor_cores: (Standalone & Mesos only) Total cores for all executors</span>
+<span class="sd"> (Default: all the available cores on the worker)</span>
<span class="sd"> :type total_executor_cores: int</span>
-<span class="sd"> :param executor_cores: (Standalone & YARN only) Number of cores per executor (Default: 2)</span>
+<span class="sd"> :param executor_cores: (Standalone & YARN only) Number of cores per</span>
+<span class="sd"> executor (Default: 2)</span>
<span class="sd"> :type executor_cores: int</span>
<span class="sd"> :param executor_memory: Memory per executor (e.g. 1000M, 2G) (Default: 1G)</span>
<span class="sd"> :type executor_memory: str</span>
@@ -333,9 +324,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -347,7 +336,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -366,6 +355,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -378,19 +368,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/spark_submit_hook.html b/_modules/airflow/contrib/hooks/spark_submit_hook.html
index 6903f5f..9e47eef 100644
--- a/_modules/airflow/contrib/hooks/spark_submit_hook.html
+++ b/_modules/airflow/contrib/hooks/spark_submit_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -197,7 +187,6 @@
<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.kubernetes</span> <span class="k">import</span> <span class="n">kube_client</span>
-<span class="kn">from</span> <span class="nn">kubernetes.client.rest</span> <span class="k">import</span> <span class="n">ApiException</span>
<div class="viewcode-block" id="SparkSubmitHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.spark_submit_hook.SparkSubmitHook">[docs]</a><span class="k">class</span> <span class="nc">SparkSubmitHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
@@ -307,6 +296,10 @@
<span class="bp">self</span><span class="o">.</span><span class="n">_connection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_resolve_connection</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_is_yarn</span> <span class="o">=</span> <span class="s1">'yarn'</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'master'</span><span class="p">]</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_is_kubernetes</span> <span class="o">=</span> <span class="s1">'k8s'</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'master'</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_is_kubernetes</span> <span class="ow">and</span> <span class="n">kube_client</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
+ <span class="s2">"</span><span class="si">{master}</span><span class="s2"> specified by kubernetes dependencies are not installed!"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_connection</span><span class="p">[</span><span class="s1">'master'</span><span class="p">]))</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_should_track_driver_status</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_resolve_should_track_driver_status</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_driver_id</span> <span class="o">=</span> <span class="kc">None</span>
@@ -730,15 +723,13 @@
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Spark on K8s killed with response: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">api_response</span><span class="p">)</span>
- <span class="k">except</span> <span class="n">ApiException</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="k">except</span> <span class="n">kube_client</span><span class="o">.</span><span class="n">ApiException</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Exception when attempting to kill Spark on K8s:"</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="n">e</span><span class="p">)</span></div>
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -750,7 +741,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -769,6 +760,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -781,19 +773,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/sqoop_hook.html b/_modules/airflow/contrib/hooks/sqoop_hook.html
index 4563030..9c13d5f 100644
--- a/_modules/airflow/contrib/hooks/sqoop_hook.html
+++ b/_modules/airflow/contrib/hooks/sqoop_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -242,7 +232,11 @@
<span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
<span class="bp">self</span><span class="o">.</span><span class="n">num_mappers</span> <span class="o">=</span> <span class="n">num_mappers</span>
<span class="bp">self</span><span class="o">.</span><span class="n">properties</span> <span class="o">=</span> <span class="n">properties</span> <span class="ow">or</span> <span class="p">{}</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Using connection to: </span><span class="si">{}</span><span class="s2">:</span><span class="si">{}</span><span class="s2">/</span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">. [...]
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s2">"Using connection to: </span><span class="si">{}</span><span class="s2">:</span><span class="si">{}</span><span class="s2">/</span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">schema</span>
+ <span class="p">)</span>
+ <span class="p">)</span>
<span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span>
@@ -266,10 +260,11 @@
<span class="sd"> """</span>
<span class="n">masked_cmd</span> <span class="o">=</span> <span class="s1">' '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cmd_mask_password</span><span class="p">(</span><span class="n">cmd</span><span class="p">))</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Executing command: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">masked_cmd</span><span class="p">))</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">sp</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">Popen</span><span class="p">(</span><span class="n">cmd</span><span class="p">,</span>
- <span class="n">stdout</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span>
- <span class="n">stderr</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">STDOUT</span><span class="p">,</span>
- <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sp</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">Popen</span><span class="p">(</span>
+ <span class="n">cmd</span><span class="p">,</span>
+ <span class="n">stdout</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span>
+ <span class="n">stderr</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">STDOUT</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="nb">iter</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sp</span><span class="o">.</span><span class="n">stdout</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">line</span><span class="o">.</span><span class="n">strip</span><span class="p">())</span>
@@ -360,7 +355,8 @@
<span class="k">if</span> <span class="n">extra_import_options</span><span class="p">:</span>
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">extra_import_options</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'--</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="p">)]</span>
- <span class="k">if</span> <span class="n">value</span><span class="p">:</span> <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="n">value</span><span class="p">]</span>
+ <span class="k">if</span> <span class="n">value</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="n">value</span><span class="p">]</span>
<span class="k">return</span> <span class="n">cmd</span>
@@ -466,7 +462,8 @@
<span class="k">if</span> <span class="n">extra_export_options</span><span class="p">:</span>
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">extra_export_options</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">'--</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="p">)]</span>
- <span class="k">if</span> <span class="n">value</span><span class="p">:</span> <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="n">value</span><span class="p">]</span>
+ <span class="k">if</span> <span class="n">value</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="n">value</span><span class="p">]</span>
<span class="c1"># The required option</span>
<span class="n">cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">"--table"</span><span class="p">,</span> <span class="n">table</span><span class="p">]</span>
@@ -517,9 +514,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -531,7 +526,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -550,6 +545,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -562,19 +558,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/ssh_hook.html b/_modules/airflow/contrib/hooks/ssh_hook.html
index 08f1831..453721f 100644
--- a/_modules/airflow/contrib/hooks/ssh_hook.html
+++ b/_modules/airflow/contrib/hooks/ssh_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -171,9 +161,6 @@
<h1>Source code for airflow.contrib.hooks.ssh_hook</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Copyright 2012-2015 Spotify AB</span>
-<span class="c1"># Ported to Airflow by Bolke de Bruin</span>
-<span class="c1">#</span>
<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
<span class="c1"># distributed with this work for additional information</span>
@@ -181,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -225,7 +212,8 @@
<span class="sd"> :type port: int</span>
<span class="sd"> :param timeout: timeout for the attempt to connect to the remote_host.</span>
<span class="sd"> :type timeout: int</span>
-<span class="sd"> :param keepalive_interval: send a keepalive packet to remote host every keepalive_interval seconds</span>
+<span class="sd"> :param keepalive_interval: send a keepalive packet to remote host every</span>
+<span class="sd"> keepalive_interval seconds</span>
<span class="sd"> :type keepalive_interval: int</span>
<span class="sd"> """</span>
@@ -407,9 +395,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -421,7 +407,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -440,6 +426,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -452,19 +439,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/vertica_hook.html b/_modules/airflow/contrib/hooks/vertica_hook.html
index 355e266..f60fa92 100644
--- a/_modules/airflow/contrib/hooks/vertica_hook.html
+++ b/_modules/airflow/contrib/hooks/vertica_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -195,9 +185,9 @@
<div class="viewcode-block" id="VerticaHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.vertica_hook.VerticaHook">[docs]</a><span class="k">class</span> <span class="nc">VerticaHook</span><span class="p">(</span><span class="n">DbApiHook</span><span class="p">):</span>
- <span class="sd">'''</span>
+ <span class="sd">"""</span>
<span class="sd"> Interact with Vertica.</span>
-<span class="sd"> '''</span>
+<span class="sd"> """</span>
<span class="n">conn_name_attr</span> <span class="o">=</span> <span class="s1">'vertica_conn_id'</span>
<span class="n">default_conn_name</span> <span class="o">=</span> <span class="s1">'vertica_default'</span>
@@ -225,9 +215,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -239,7 +227,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -258,6 +246,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -270,19 +259,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/wasb_hook.html b/_modules/airflow/contrib/hooks/wasb_hook.html
index 605fbf3..e98b5cf 100644
--- a/_modules/airflow/contrib/hooks/wasb_hook.html
+++ b/_modules/airflow/contrib/hooks/wasb_hook.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -322,9 +312,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -336,7 +324,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -355,6 +343,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -367,19 +356,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/winrm_hook.html b/_modules/airflow/contrib/hooks/winrm_hook.html
new file mode 100644
index 0000000..c89917c
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/winrm_hook.html
@@ -0,0 +1,355 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.winrm_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.winrm_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.winrm_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+<span class="kn">import</span> <span class="nn">getpass</span>
+<span class="kn">from</span> <span class="nn">winrm.protocol</span> <span class="k">import</span> <span class="n">Protocol</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+
+<div class="viewcode-block" id="WinRMHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.winrm_hook.WinRMHook">[docs]</a><span class="k">class</span> <span class="nc">WinRMHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+
+ <span class="sd">"""</span>
+<span class="sd"> Hook for winrm remote execution using pywinrm.</span>
+
+<span class="sd"> :param ssh_conn_id: connection id from airflow Connections from where all</span>
+<span class="sd"> the required parameters can be fetched like username, password or key_file.</span>
+<span class="sd"> Thought the priority is given to the param passed during init</span>
+<span class="sd"> :type ssh_conn_id: str</span>
+<span class="sd"> :param remote_host: remote host to connect</span>
+<span class="sd"> :type remote_host: str</span>
+<span class="sd"> :param username: username to connect to the remote_host</span>
+<span class="sd"> :type username: str</span>
+<span class="sd"> :param password: password of the username to connect to the remote_host</span>
+<span class="sd"> :type password: str</span>
+<span class="sd"> :param key_file: key file to use to connect to the remote_host.</span>
+<span class="sd"> :type key_file: str</span>
+<span class="sd"> :param timeout: timeout for the attempt to connect to the remote_host.</span>
+<span class="sd"> :type timeout: int</span>
+<span class="sd"> :param keepalive_interval: send a keepalive packet to remote host</span>
+<span class="sd"> every keepalive_interval seconds</span>
+<span class="sd"> :type keepalive_interval: int</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">ssh_conn_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">remote_host</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">username</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">password</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">key_file</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">timeout</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span>
+ <span class="n">keepalive_interval</span><span class="o">=</span><span class="mi">30</span>
+ <span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">WinRMHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">ssh_conn_id</span><span class="p">)</span>
+ <span class="c1"># TODO make new win rm connection class</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span> <span class="o">=</span> <span class="n">ssh_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span> <span class="o">=</span> <span class="n">remote_host</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="o">=</span> <span class="n">username</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="o">=</span> <span class="n">password</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span> <span class="o">=</span> <span class="n">key_file</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span> <span class="o">=</span> <span class="n">timeout</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">keepalive_interval</span> <span class="o">=</span> <span class="n">keepalive_interval</span>
+ <span class="c1"># Default values, overridable from Connection</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">compress</span> <span class="o">=</span> <span class="kc">True</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">no_host_key_check</span> <span class="o">=</span> <span class="kc">True</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">winrm_protocol</span> <span class="o">=</span> <span class="kc">None</span>
+
+ <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'Creating WinRM client for conn_id: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span>
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">extra_options</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span> <span class="o">=</span> <span class="n">extra_options</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"key_file"</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="s2">"timeout"</span> <span class="ow">in</span> <span class="n">extra_options</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">extra_options</span><span class="p">[</span><span class="s2">"timeout"</span><span class="p">],</span> <span class="mi">10</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="s2">"compress"</span> <span class="ow">in</span> <span class="n">extra_options</span> \
+ <span class="ow">and</span> <span class="n">extra_options</span><span class="p">[</span><span class="s2">"compress"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'false'</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">compress</span> <span class="o">=</span> <span class="kc">False</span>
+ <span class="k">if</span> <span class="s2">"no_host_key_check"</span> <span class="ow">in</span> <span class="n">extra_options</span> \
+ <span class="ow">and</span> <span class="n">extra_options</span><span class="p">[</span><span class="s2">"no_host_key_check"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'false'</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">no_host_key_check</span> <span class="o">=</span> <span class="kc">False</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Missing required param: remote_host"</span><span class="p">)</span>
+
+ <span class="c1"># Auto detecting username values from system</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
+ <span class="s2">"username to ssh to host: </span><span class="si">%s</span><span class="s2"> is not specified for connection id"</span>
+ <span class="s2">" </span><span class="si">%s</span><span class="s2">. Using system's default provided by getpass.getuser()"</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span>
+ <span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="o">=</span> <span class="n">getpass</span><span class="o">.</span><span class="n">getuser</span><span class="p">()</span>
+
+ <span class="k">try</span><span class="p">:</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span><span class="o">.</span><span class="n">strip</span><span class="p">():</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">winrm_protocol</span> <span class="o">=</span> <span class="n">Protocol</span><span class="p">(</span>
+ <span class="c1"># TODO pass in port from ssh conn</span>
+ <span class="n">endpoint</span><span class="o">=</span><span class="s1">'http://'</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span> <span class="o">+</span> <span class="s1">':5985/wsman'</span><span class="p">,</span>
+ <span class="c1"># TODO get cert transport working</span>
+ <span class="c1"># transport='certificate',</span>
+ <span class="n">transport</span><span class="o">=</span><span class="s1">'plaintext'</span><span class="p">,</span>
+ <span class="c1"># cert_pem=r'publickey.pem',</span>
+ <span class="c1"># cert_key_pem=r'dev.pem',</span>
+ <span class="n">read_timeout_sec</span><span class="o">=</span><span class="mi">70</span><span class="p">,</span>
+ <span class="n">operation_timeout_sec</span><span class="o">=</span><span class="mi">60</span><span class="p">,</span>
+ <span class="n">username</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">,</span>
+ <span class="n">password</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">password</span><span class="p">,</span>
+ <span class="n">server_cert_validation</span><span class="o">=</span><span class="s1">'ignore'</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Opening WinRM shell"</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">winrm_protocol</span><span class="o">.</span><span class="n">open_shell</span><span class="p">()</span>
+
+ <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
+ <span class="s2">"Error connecting to host: </span><span class="si">%s</span><span class="s2">, error: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span> <span class="n">error</span>
+ <span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../../',
+ VERSION:'',
+ LANGUAGE:'None',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/kubernetes/secret.html b/_modules/airflow/contrib/kubernetes/secret.html
index c5785ed..4d77038 100644
--- a/_modules/airflow/contrib/kubernetes/secret.html
+++ b/_modules/airflow/contrib/kubernetes/secret.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -213,9 +203,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -227,7 +215,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -246,6 +234,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -258,19 +247,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/operators/awsbatch_operator.html b/_modules/airflow/contrib/operators/awsbatch_operator.html
index 33e1e8b..9f65bb3 100644
--- a/_modules/airflow/contrib/operators/awsbatch_operator.html
+++ b/_modules/airflow/contrib/operators/awsbatch_operator.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -204,21 +194,27 @@
<span class="sd">"""</span>
<span class="sd"> Execute a job on AWS Batch Service</span>
+<span class="sd"> .. warning: the queue parameter was renamed to job_queue to segreggate the</span>
+<span class="sd"> internal CeleryExecutor queue from the AWS Batch internal queue.</span>
+
<span class="sd"> :param job_name: the name for the job that will run on AWS Batch</span>
<span class="sd"> :type job_name: str</span>
<span class="sd"> :param job_definition: the job definition name on AWS Batch</span>
<span class="sd"> :type job_definition: str</span>
-<span class="sd"> :param queue: the queue name on AWS Batch</span>
-<span class="sd"> :type queue: str</span>
-<span class="sd"> :param: overrides: the same parameter that boto3 will receive on containerOverrides:</span>
+<span class="sd"> :param job_queue: the queue name on AWS Batch</span>
+<span class="sd"> :type job_queue: str</span>
+<span class="sd"> :param: overrides: the same parameter that boto3 will receive on</span>
+<span class="sd"> containerOverrides (templated):</span>
<span class="sd"> http://boto3.readthedocs.io/en/latest/reference/services/batch.html#submit_job</span>
<span class="sd"> :type: overrides: dict</span>
-<span class="sd"> :param max_retries: exponential backoff retries while waiter is not merged</span>
+<span class="sd"> :param max_retries: exponential backoff retries while waiter is not merged, 4200 = 48 hours</span>
<span class="sd"> :type max_retries: int</span>
<span class="sd"> :param aws_conn_id: connection id of AWS credentials / region name. If None,</span>
-<span class="sd"> credential boto3 strategy will be used (http://boto3.readthedocs.io/en/latest/guide/configuration.html).</span>
+<span class="sd"> credential boto3 strategy will be used</span>
+<span class="sd"> (http://boto3.readthedocs.io/en/latest/guide/configuration.html).</span>
<span class="sd"> :type aws_conn_id: str</span>
-<span class="sd"> :param region_name: region name to use in AWS Hook. Override the region_name in connection (if provided)</span>
+<span class="sd"> :param region_name: region name to use in AWS Hook.</span>
+<span class="sd"> Override the region_name in connection (if provided)</span>
<span class="sd"> """</span>
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#c3dae0'</span>
@@ -227,7 +223,7 @@
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'overrides'</span><span class="p">,)</span>
<span class="nd">@apply_defaults</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">job_name</span><span class="p">,</span> <span class="n">job_definition</span><span class="p">,</span> <span class="n">queue</span><span class="p">,</span> <span class="n">overrides</span><span class="p">,</span> <span class="n">max_retries</span><span class="o">=</span><span class="mi">288</span><span class="p">,</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">job_name</span><span class="p">,</span> <span class="n">job_definition</span><span class="p">,</span> <span class="n">job_queue</span><span class="p">,</span> <span class="n">overrides</span><span class="p">,</span> <span class="n">max_retries</span><span class="o">=</span><span class="mi">4200</span><span class="p">,</span>
<span class="n">aws_conn_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">AWSBatchOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
@@ -235,7 +231,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span> <span class="o">=</span> <span class="n">aws_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">region_name</span> <span class="o">=</span> <span class="n">region_name</span>
<span class="bp">self</span><span class="o">.</span><span class="n">job_definition</span> <span class="o">=</span> <span class="n">job_definition</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">queue</span> <span class="o">=</span> <span class="n">queue</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job_queue</span> <span class="o">=</span> <span class="n">job_queue</span>
<span class="bp">self</span><span class="o">.</span><span class="n">overrides</span> <span class="o">=</span> <span class="n">overrides</span>
<span class="bp">self</span><span class="o">.</span><span class="n">max_retries</span> <span class="o">=</span> <span class="n">max_retries</span>
@@ -247,7 +243,7 @@
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
<span class="s1">'Running AWS Batch Job - Job definition: </span><span class="si">%s</span><span class="s1"> - on queue </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">job_definition</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">queue</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job_definition</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_queue</span>
<span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'AWSBatchOperator overrides: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">overrides</span><span class="p">)</span>
@@ -259,7 +255,7 @@
<span class="k">try</span><span class="p">:</span>
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">submit_job</span><span class="p">(</span>
<span class="n">jobName</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">,</span>
- <span class="n">jobQueue</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="p">,</span>
+ <span class="n">jobQueue</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">job_queue</span><span class="p">,</span>
<span class="n">jobDefinition</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">job_definition</span><span class="p">,</span>
<span class="n">containerOverrides</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">overrides</span><span class="p">)</span>
@@ -296,14 +292,15 @@
<span class="n">retry</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">retries</span> <span class="o">=</span> <span class="mi">0</span>
- <span class="k">while</span> <span class="n">retries</span> <span class="o"><</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_retries</span> <span class="ow">or</span> <span class="n">retry</span><span class="p">:</span>
+ <span class="k">while</span> <span class="n">retries</span> <span class="o"><</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_retries</span> <span class="ow">and</span> <span class="n">retry</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'AWS Batch retry in the next </span><span class="si">%s</span><span class="s1"> seconds'</span><span class="p">,</span> <span class="n">retries</span><span class="p">)</span>
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">describe_jobs</span><span class="p">(</span>
<span class="n">jobs</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">jobId</span><span class="p">]</span>
<span class="p">)</span>
<span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s1">'jobs'</span><span class="p">][</span><span class="o">-</span><span class="mi">1</span><span class="p">][</span><span class="s1">'status'</span><span class="p">]</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'SUCCEEDED'</span><span class="p">,</span> <span class="s1">'FAILED'</span><span class="p">]:</span>
<span class="n">retry</span> <span class="o">=</span> <span class="kc">False</span>
- <span class="n">sleep</span><span class="p">(</span><span class="nb">pow</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="n">retries</span><span class="p">)</span> <span class="o">*</span> <span class="mi">100</span><span class="p">)</span>
+ <span class="n">sleep</span><span class="p">(</span> <span class="mi">1</span> <span class="o">+</span> <span class="nb">pow</span><span class="p">(</span><span class="n">retries</span> <span class="o">*</span> <span class="mf">0.1</span><span class="p">,</span> <span class="mi">2</span><span class="p">))</span>
<span class="n">retries</span> <span class="o">+=</span> <span class="mi">1</span>
<span class="k">def</span> <span class="nf">_check_success_task</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
@@ -321,10 +318,12 @@
<span class="k">for</span> <span class="n">container</span> <span class="ow">in</span> <span class="n">containers</span><span class="p">:</span>
<span class="k">if</span> <span class="p">(</span><span class="n">job</span><span class="p">[</span><span class="s1">'status'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'FAILED'</span> <span class="ow">or</span>
<span class="n">container</span><span class="p">[</span><span class="s1">'container'</span><span class="p">][</span><span class="s1">'exitCode'</span><span class="p">]</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">):</span>
- <span class="nb">print</span><span class="p">(</span><span class="s2">"@@@@"</span><span class="p">)</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'This containers encounter an error during execution </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">job</span><span class="p">))</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'This containers encounter an error during '</span>
+ <span class="s1">'execution </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">job</span><span class="p">))</span>
<span class="k">elif</span> <span class="n">job</span><span class="p">[</span><span class="s1">'status'</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="s1">'SUCCEEDED'</span><span class="p">:</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'This task is still pending </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">job</span><span class="p">[</span><span class="s1">'status'</span><span class="p">]))</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'This task is still pending </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">job</span><span class="p">[</span><span class="s1">'status'</span><span class="p">]))</span>
<span class="k">def</span> <span class="nf">get_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="n">AwsHook</span><span class="p">(</span>
@@ -340,9 +339,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -354,7 +351,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -373,6 +370,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -385,19 +383,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/operators/bigquery_check_operator.html b/_modules/airflow/contrib/operators/bigquery_check_operator.html
index 1be7f24..9202dec 100644
--- a/_modules/airflow/contrib/operators/bigquery_check_operator.html
+++ b/_modules/airflow/contrib/operators/bigquery_check_operator.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -189,7 +179,8 @@
<span class="c1"># under the License.</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="k">import</span> <span class="n">BigQueryHook</span>
-<span class="kn">from</span> <span class="nn">airflow.operators.check_operator</span> <span class="k">import</span> <span class="n">CheckOperator</span><span class="p">,</span> <span class="n">ValueCheckOperator</span><span class="p">,</span> <span class="n">IntervalCheckOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.operators.check_operator</span> <span class="k">import</span> \
+ <span class="n">CheckOperator</span><span class="p">,</span> <span class="n">ValueCheckOperator</span><span class="p">,</span> <span class="n">IntervalCheckOperator</span>
<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
@@ -302,9 +293,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -316,7 +305,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -335,6 +324,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -347,19 +337,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/operators/bigquery_get_data.html b/_modules/airflow/contrib/operators/bigquery_get_data.html
index 9d28a70..abff695 100644
--- a/_modules/airflow/contrib/operators/bigquery_get_data.html
+++ b/_modules/airflow/contrib/operators/bigquery_get_data.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -221,12 +211,12 @@
<span class="sd"> bigquery_conn_id='airflow-service-account'</span>
<span class="sd"> )</span>
-<span class="sd"> :param dataset_id: The dataset ID of the requested table.</span>
+<span class="sd"> :param dataset_id: The dataset ID of the requested table. (templated)</span>
<span class="sd"> :type destination_dataset_table: string</span>
-<span class="sd"> :param table_id: The table ID of the requested table.</span>
+<span class="sd"> :param table_id: The table ID of the requested table. (templated)</span>
<span class="sd"> :type table_id: string</span>
<span class="sd"> :param max_results: The maximum number of records (rows) to be fetched</span>
-<span class="sd"> from the table.</span>
+<span class="sd"> from the table. (templated)</span>
<span class="sd"> :type max_results: string</span>
<span class="sd"> :param selected_fields: List of fields to return (comma-separated). If</span>
<span class="sd"> unspecified, all fields are returned.</span>
@@ -288,9 +278,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -302,7 +290,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -321,6 +309,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -333,19 +322,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/operators/bigquery_operator.html b/_modules/airflow/contrib/operators/bigquery_operator.html
index 4055654..f4a00d9 100644
--- a/_modules/airflow/contrib/operators/bigquery_operator.html
+++ b/_modules/airflow/contrib/operators/bigquery_operator.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -200,13 +190,18 @@
<span class="sd">"""</span>
<span class="sd"> Executes BigQuery SQL queries in a specific BigQuery database</span>
-<span class="sd"> :param bql: the sql code to be executed</span>
+<span class="sd"> :param bql: (Deprecated. Use `sql` parameter instead) the sql code to be</span>
+<span class="sd"> executed (templated)</span>
<span class="sd"> :type bql: Can receive a str representing a sql statement,</span>
<span class="sd"> a list of str (sql statements), or reference to a template file.</span>
-<span class="sd"> Template reference are recognized by str ending in '.sql'</span>
+<span class="sd"> Template reference are recognized by str ending in '.sql'.</span>
+<span class="sd"> :param sql: the sql code to be executed (templated)</span>
+<span class="sd"> :type sql: Can receive a str representing a sql statement,</span>
+<span class="sd"> a list of str (sql statements), or reference to a template file.</span>
+<span class="sd"> Template reference are recognized by str ending in '.sql'.</span>
<span class="sd"> :param destination_dataset_table: A dotted</span>
<span class="sd"> (<project>.|<project>:)<dataset>.<table> that, if set, will store the results</span>
-<span class="sd"> of the query.</span>
+<span class="sd"> of the query. (templated)</span>
<span class="sd"> :type destination_dataset_table: string</span>
<span class="sd"> :param write_disposition: Specifies the action that occurs if the destination table</span>
<span class="sd"> already exists. (default: 'WRITE_EMPTY')</span>
@@ -247,15 +242,25 @@
<span class="sd"> :param query_params: a dictionary containing query parameter types and</span>
<span class="sd"> values, passed to BigQuery.</span>
<span class="sd"> :type query_params: dict</span>
-
+<span class="sd"> :param priority: Specifies a priority for the query.</span>
+<span class="sd"> Possible values include INTERACTIVE and BATCH.</span>
+<span class="sd"> The default value is INTERACTIVE.</span>
+<span class="sd"> :type priority: string</span>
+<span class="sd"> :param time_partitioning: configure optional time partitioning fields i.e.</span>
+<span class="sd"> partition by field, type and</span>
+<span class="sd"> expiration as per API specifications. Note that 'field' is not available in</span>
+<span class="sd"> conjunction with dataset.table$partition.</span>
+<span class="sd"> :type time_partitioning: dict</span>
<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bql'</span><span class="p">,</span> <span class="s1">'destination_dataset_table'</span><span class="p">)</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bql'</span><span class="p">,</span> <span class="s1">'sql'</span><span class="p">,</span> <span class="s1">'destination_dataset_table'</span><span class="p">)</span>
<span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,</span> <span class="p">)</span>
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#e4f0e8'</span>
<span class="nd">@apply_defaults</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
- <span class="n">bql</span><span class="p">,</span>
+ <span class="n">bql</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">sql</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">destination_dataset_table</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">write_disposition</span><span class="o">=</span><span class="s1">'WRITE_EMPTY'</span><span class="p">,</span>
<span class="n">allow_large_results</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
@@ -270,10 +275,12 @@
<span class="n">schema_update_options</span><span class="o">=</span><span class="p">(),</span>
<span class="n">query_params</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">priority</span><span class="o">=</span><span class="s1">'INTERACTIVE'</span><span class="p">,</span>
+ <span class="n">time_partitioning</span><span class="o">=</span><span class="p">{},</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span>
<span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">BigQueryOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bql</span> <span class="o">=</span> <span class="n">bql</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span> <span class="k">if</span> <span class="n">sql</span> <span class="k">else</span> <span class="n">bql</span>
<span class="bp">self</span><span class="o">.</span><span class="n">destination_dataset_table</span> <span class="o">=</span> <span class="n">destination_dataset_table</span>
<span class="bp">self</span><span class="o">.</span><span class="n">write_disposition</span> <span class="o">=</span> <span class="n">write_disposition</span>
<span class="bp">self</span><span class="o">.</span><span class="n">create_disposition</span> <span class="o">=</span> <span class="n">create_disposition</span>
@@ -289,10 +296,25 @@
<span class="bp">self</span><span class="o">.</span><span class="n">query_params</span> <span class="o">=</span> <span class="n">query_params</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bq_cursor</span> <span class="o">=</span> <span class="kc">None</span>
<span class="bp">self</span><span class="o">.</span><span class="n">priority</span> <span class="o">=</span> <span class="n">priority</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">time_partitioning</span> <span class="o">=</span> <span class="n">time_partitioning</span>
+
+ <span class="c1"># TODO remove `bql` in Airflow 2.0</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">bql</span><span class="p">:</span>
+ <span class="kn">import</span> <span class="nn">warnings</span>
+ <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s1">'Deprecated parameter `bql` used in Task id: </span><span class="si">{}</span><span class="s1">. '</span>
+ <span class="s1">'Use `sql` parameter instead to pass the sql to be '</span>
+ <span class="s1">'executed. `bql` parameter is deprecated and '</span>
+ <span class="s1">'will be removed in a future version of '</span>
+ <span class="s1">'Airflow.'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">),</span>
+ <span class="n">category</span><span class="o">=</span><span class="ne">DeprecationWarning</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">'</span><span class="si">{}</span><span class="s1"> missing 1 required positional '</span>
+ <span class="s1">'argument: `sql`'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">bq_cursor</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bql</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span>
<span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
<span class="n">use_legacy_sql</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span><span class="p">,</span>
@@ -300,7 +322,7 @@
<span class="n">conn</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bq_cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bq_cursor</span><span class="o">.</span><span class="n">run_query</span><span class="p">(</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">bql</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">,</span>
<span class="n">destination_dataset_table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">destination_dataset_table</span><span class="p">,</span>
<span class="n">write_disposition</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">write_disposition</span><span class="p">,</span>
<span class="n">allow_large_results</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">allow_large_results</span><span class="p">,</span>
@@ -311,7 +333,9 @@
<span class="n">create_disposition</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">create_disposition</span><span class="p">,</span>
<span class="n">query_params</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">query_params</span><span class="p">,</span>
<span class="n">schema_update_options</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">schema_update_options</span><span class="p">,</span>
- <span class="n">priority</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">priority</span><span class="p">)</span>
+ <span class="n">priority</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">priority</span><span class="p">,</span>
+ <span class="n">time_partitioning</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">time_partitioning</span>
+ <span class="p">)</span>
<span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">BigQueryOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">on_kill</span><span class="p">()</span>
@@ -331,11 +355,11 @@
<span class="sd"> Google cloud storage must be a JSON file with the schema fields in it.</span>
<span class="sd"> You can also create a table without schema.</span>
-<span class="sd"> :param project_id: The project to create the table into.</span>
+<span class="sd"> :param project_id: The project to create the table into. (templated)</span>
<span class="sd"> :type project_id: string</span>
-<span class="sd"> :param dataset_id: The dataset to create the table into.</span>
+<span class="sd"> :param dataset_id: The dataset to create the table into. (templated)</span>
<span class="sd"> :type dataset_id: string</span>
-<span class="sd"> :param table_id: The Name of the table to be created.</span>
+<span class="sd"> :param table_id: The Name of the table to be created. (templated)</span>
<span class="sd"> :type table_id: string</span>
<span class="sd"> :param schema_fields: If set, the schema field list as defined here:</span>
<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load.schema</span>
@@ -346,7 +370,8 @@
<span class="sd"> {"name": "salary", "type": "INTEGER", "mode": "NULLABLE"}]</span>
<span class="sd"> :type schema_fields: list</span>
-<span class="sd"> :param gcs_schema_object: Full path to the JSON file containing schema. For</span>
+<span class="sd"> :param gcs_schema_object: Full path to the JSON file containing</span>
+<span class="sd"> schema (templated). For</span>
<span class="sd"> example: ``gs://test-bucket/dir1/dir2/employee_schema.json``</span>
<span class="sd"> :type gcs_schema_object: string</span>
<span class="sd"> :param time_partitioning: configure optional time partitioning fields i.e.</span>
@@ -473,14 +498,15 @@
<span class="sd"> point the operator to a Google cloud storage object name. The object in</span>
<span class="sd"> Google cloud storage must be a JSON file with the schema fields in it.</span>
-<span class="sd"> :param bucket: The bucket to point the external table to.</span>
+<span class="sd"> :param bucket: The bucket to point the external table to. (templated)</span>
<span class="sd"> :type bucket: string</span>
-<span class="sd"> :param source_objects: List of Google cloud storage URIs to point table to.</span>
+<span class="sd"> :param source_objects: List of Google cloud storage URIs to point</span>
+<span class="sd"> table to. (templated)</span>
<span class="sd"> If source_format is 'DATASTORE_BACKUP', the list must only contain a single URI.</span>
<span class="sd"> :type object: list</span>
<span class="sd"> :param destination_project_dataset_table: The dotted (<project>.)<dataset>.<table></span>
-<span class="sd"> BigQuery table to load data into. If <project> is not included, project will</span>
-<span class="sd"> be the project defined in the connection json.</span>
+<span class="sd"> BigQuery table to load data into (templated). If <project> is not included,</span>
+<span class="sd"> project will be the project defined in the connection json.</span>
<span class="sd"> :type destination_project_dataset_table: string</span>
<span class="sd"> :param schema_fields: If set, the schema field list as defined here:</span>
<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load.schema</span>
@@ -493,7 +519,7 @@
<span class="sd"> Should not be set when source_format is 'DATASTORE_BACKUP'.</span>
<span class="sd"> :type schema_fields: list</span>
<span class="sd"> :param schema_object: If set, a GCS object path pointing to a .json file that</span>
-<span class="sd"> contains the schema for the table.</span>
+<span class="sd"> contains the schema for the table. (templated)</span>
<span class="sd"> :param schema_object: string</span>
<span class="sd"> :param source_format: File format of the data.</span>
<span class="sd"> :type source_format: string</span>
@@ -619,9 +645,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -633,7 +657,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -652,6 +676,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -664,19 +689,13 @@
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/operators/bigquery_table_delete_operator.html b/_modules/airflow/contrib/operators/bigquery_table_delete_operator.html
index 94bc9dd..8b3a424 100644
--- a/_modules/airflow/contrib/operators/bigquery_table_delete_operator.html
+++ b/_modules/airflow/contrib/operators/bigquery_table_delete_operator.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
- <link rel="up" title="Module code" href="../../../index.html"/>
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -199,7 +189,7 @@
<span class="sd"> :param deletion_dataset_table: A dotted</span>
<span class="sd"> (<project>.|<project>:)<dataset>.<table> that indicates which table</span>
-<span class="sd"> will be deleted.</span>
+<span class="sd"> will be deleted. (templated)</span>
<span class="sd"> :type deletion_dataset_table: string</span>
<span class="sd"> :param bigquery_conn_id: reference to a specific BigQuery hook.</span>
<span class="sd"> :type bigquery_conn_id: string</span>
@@ -238,9 +228,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -252,7 +240,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -271,6 +259,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../../',
VERSION:'',
+ LANGUAGE:'None',
... 55865 lines suppressed ...