You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by as...@apache.org on 2019/11/27 19:10:25 UTC
[airflow-site] 12/30: Docs from 1.10.1
This is an automated email from the ASF dual-hosted git repository.
ash pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/airflow-site.git
commit 1f06fa0e0ead70d24f3c87326734aec89930207d
Author: Ash Berlin-Taylor <as...@firemirror.com>
AuthorDate: Thu Nov 29 15:57:25 2018 +0000
Docs from 1.10.1
---
_images/airflow.gif | Bin 622963 -> 622832 bytes
.../airflow/contrib/executors/mesos_executor.html | 548 ----
.../airflow/contrib/hooks/aws_dynamodb_hook.html | 21 +-
.../aws_firehose_hook.html} | 86 +-
_modules/airflow/contrib/hooks/aws_hook.html | 104 +-
.../airflow/contrib/hooks/aws_lambda_hook.html | 21 +-
.../contrib/hooks/azure_data_lake_hook.html | 21 +-
.../contrib/hooks/azure_fileshare_hook.html | 21 +-
_modules/airflow/contrib/hooks/bigquery_hook.html | 139 +-
_modules/airflow/contrib/hooks/cassandra_hook.html | 21 +-
_modules/airflow/contrib/hooks/cloudant_hook.html | 21 +-
.../airflow/contrib/hooks/databricks_hook.html | 87 +-
_modules/airflow/contrib/hooks/datadog_hook.html | 21 +-
_modules/airflow/contrib/hooks/datastore_hook.html | 21 +-
.../contrib/hooks/discord_webhook_hook.html | 21 +-
_modules/airflow/contrib/hooks/emr_hook.html | 36 +-
_modules/airflow/contrib/hooks/fs_hook.html | 21 +-
_modules/airflow/contrib/hooks/ftp_hook.html | 91 +-
.../airflow/contrib/hooks/gcp_api_base_hook.html | 23 +-
.../airflow/contrib/hooks/gcp_container_hook.html | 25 +-
.../airflow/contrib/hooks/gcp_dataflow_hook.html | 128 +-
.../airflow/contrib/hooks/gcp_dataproc_hook.html | 21 +-
.../airflow/contrib/hooks/gcp_function_hook.html | 406 +++
.../airflow/contrib/hooks/gcp_mlengine_hook.html | 32 +-
.../airflow/contrib/hooks/gcp_pubsub_hook.html | 21 +-
_modules/airflow/contrib/hooks/gcp_sql_hook.html | 474 +++
_modules/airflow/contrib/hooks/gcs_hook.html | 60 +-
_modules/airflow/contrib/hooks/imap_hook.html | 491 +++
_modules/airflow/contrib/hooks/jenkins_hook.html | 21 +-
_modules/airflow/contrib/hooks/jira_hook.html | 21 +-
_modules/airflow/contrib/hooks/mongo_hook.html | 21 +-
_modules/airflow/contrib/hooks/pinot_hook.html | 21 +-
_modules/airflow/contrib/hooks/qubole_hook.html | 21 +-
_modules/airflow/contrib/hooks/redis_hook.html | 21 +-
_modules/airflow/contrib/hooks/redshift_hook.html | 21 +-
_modules/airflow/contrib/hooks/sagemaker_hook.html | 972 ++++++
_modules/airflow/contrib/hooks/segment_hook.html | 21 +-
_modules/airflow/contrib/hooks/sftp_hook.html | 98 +-
.../airflow/contrib/hooks/slack_webhook_hook.html | 21 +-
_modules/airflow/contrib/hooks/snowflake_hook.html | 21 +-
.../airflow/contrib/hooks/spark_jdbc_hook.html | 21 +-
_modules/airflow/contrib/hooks/spark_sql_hook.html | 21 +-
.../airflow/contrib/hooks/spark_submit_hook.html | 21 +-
_modules/airflow/contrib/hooks/sqoop_hook.html | 21 +-
_modules/airflow/contrib/hooks/ssh_hook.html | 315 +-
_modules/airflow/contrib/hooks/vertica_hook.html | 21 +-
_modules/airflow/contrib/hooks/wasb_hook.html | 64 +-
_modules/airflow/contrib/hooks/winrm_hook.html | 21 +-
_modules/airflow/contrib/kubernetes/secret.html | 21 +-
.../contrib/operators/awsbatch_operator.html | 47 +-
.../contrib/operators/bigquery_check_operator.html | 71 +-
.../contrib/operators/bigquery_get_data.html | 25 +-
.../contrib/operators/bigquery_operator.html | 202 +-
.../operators/bigquery_table_delete_operator.html | 25 +-
.../contrib/operators/bigquery_to_bigquery.html | 33 +-
.../airflow/contrib/operators/bigquery_to_gcs.html | 34 +-
.../contrib/operators/cassandra_to_gcs.html | 25 +-
.../contrib/operators/databricks_operator.html | 37 +-
.../contrib/operators/dataflow_operator.html | 104 +-
.../contrib/operators/dataproc_operator.html | 130 +-
.../operators/datastore_export_operator.html | 25 +-
.../operators/datastore_import_operator.html | 25 +-
.../operators/discord_webhook_operator.html | 21 +-
.../airflow/contrib/operators/druid_operator.html | 25 +-
.../airflow/contrib/operators/ecs_operator.html | 52 +-
.../contrib/operators/emr_add_steps_operator.html | 25 +-
.../operators/emr_create_job_flow_operator.html | 25 +-
.../operators/emr_terminate_job_flow_operator.html | 25 +-
.../airflow/contrib/operators/file_to_gcs.html | 34 +-
.../airflow/contrib/operators/file_to_wasb.html | 21 +-
.../contrib/operators/gcp_compute_operator.html | 394 +++
.../contrib/operators/gcp_container_operator.html | 29 +-
.../contrib/operators/gcp_function_operator.html | 522 ++++
.../contrib/operators/gcp_sql_operator.html | 737 +++++
.../contrib/operators/gcs_download_operator.html | 25 +-
.../contrib/operators/gcs_list_operator.html | 25 +-
.../airflow/contrib/operators/gcs_operator.html | 25 +-
_modules/airflow/contrib/operators/gcs_to_bq.html | 25 +-
_modules/airflow/contrib/operators/gcs_to_gcs.html | 25 +-
_modules/airflow/contrib/operators/gcs_to_s3.html | 25 +-
.../contrib/operators/hipchat_operator.html | 25 +-
.../contrib/operators/hive_to_dynamodb.html | 25 +-
.../operators/jenkins_job_trigger_operator.html | 25 +-
.../airflow/contrib/operators/jira_operator.html | 25 +-
.../contrib/operators/kubernetes_pod_operator.html | 51 +-
.../contrib/operators/mlengine_operator.html | 41 +-
.../airflow/contrib/operators/mongo_to_s3.html | 21 +-
.../airflow/contrib/operators/mysql_to_gcs.html | 25 +-
.../operators/postgres_to_gcs_operator.html | 25 +-
.../airflow/contrib/operators/pubsub_operator.html | 41 +-
.../contrib/operators/qubole_check_operator.html | 29 +-
.../airflow/contrib/operators/qubole_operator.html | 29 +-
.../contrib/operators/s3_list_operator.html | 25 +-
.../contrib/operators/s3_to_gcs_operator.html | 25 +-
.../contrib/operators/sagemaker_base_operator.html | 311 ++
.../sagemaker_endpoint_config_operator.html} | 96 +-
.../operators/sagemaker_endpoint_operator.html | 362 +++
...operator.html => sagemaker_model_operator.html} | 98 +-
.../operators/sagemaker_training_operator.html | 310 ++
.../operators/sagemaker_transform_operator.html | 336 ++
.../operators/sagemaker_tuning_operator.html | 310 ++
.../operators/segment_track_event_operator.html | 25 +-
.../airflow/contrib/operators/sftp_operator.html | 57 +-
.../contrib/operators/slack_webhook_operator.html | 25 +-
.../contrib/operators/snowflake_operator.html | 25 +-
.../contrib/operators/spark_jdbc_operator.html | 25 +-
.../contrib/operators/spark_sql_operator.html | 25 +-
.../contrib/operators/spark_submit_operator.html | 25 +-
.../airflow/contrib/operators/sqoop_operator.html | 25 +-
.../airflow/contrib/operators/ssh_operator.html | 164 +-
.../contrib/operators/vertica_operator.html | 25 +-
.../airflow/contrib/operators/vertica_to_hive.html | 25 +-
.../airflow/contrib/operators/winrm_operator.html | 25 +-
.../sensors/aws_redshift_cluster_sensor.html | 21 +-
_modules/airflow/contrib/sensors/bash_sensor.html | 21 +-
.../airflow/contrib/sensors/bigquery_sensor.html | 21 +-
...redis_key_sensor.html => cassandra_sensor.html} | 78 +-
.../airflow/contrib/sensors/datadog_sensor.html | 21 +-
.../airflow/contrib/sensors/emr_base_sensor.html | 21 +-
.../contrib/sensors/emr_job_flow_sensor.html | 21 +-
.../airflow/contrib/sensors/emr_step_sensor.html | 25 +-
_modules/airflow/contrib/sensors/file_sensor.html | 21 +-
_modules/airflow/contrib/sensors/ftp_sensor.html | 21 +-
_modules/airflow/contrib/sensors/gcs_sensor.html | 21 +-
_modules/airflow/contrib/sensors/hdfs_sensor.html | 21 +-
_modules/airflow/contrib/sensors/jira_sensor.html | 21 +-
.../airflow/contrib/sensors/pubsub_sensor.html | 21 +-
.../airflow/contrib/sensors/qubole_sensor.html | 21 +-
.../airflow/contrib/sensors/redis_key_sensor.html | 21 +-
...base_sensor.html => sagemaker_base_sensor.html} | 73 +-
..._sensor.html => sagemaker_endpoint_sensor.html} | 74 +-
.../contrib/sensors/sagemaker_training_sensor.html | 313 ++
...sensor.html => sagemaker_transform_sensor.html} | 81 +-
...ow_sensor.html => sagemaker_tuning_sensor.html} | 75 +-
_modules/airflow/contrib/sensors/sftp_sensor.html | 21 +-
_modules/airflow/contrib/sensors/wasb_sensor.html | 21 +-
_modules/airflow/executors/celery_executor.html | 27 +-
_modules/airflow/executors/local_executor.html | 21 +-
.../airflow/executors/sequential_executor.html | 21 +-
_modules/airflow/hooks/S3_hook.html | 164 +-
_modules/airflow/hooks/dbapi_hook.html | 21 +-
_modules/airflow/hooks/docker_hook.html | 21 +-
_modules/airflow/hooks/druid_hook.html | 35 +-
_modules/airflow/hooks/hdfs_hook.html | 21 +-
_modules/airflow/hooks/hive_hooks.html | 21 +-
_modules/airflow/hooks/http_hook.html | 33 +-
_modules/airflow/hooks/jdbc_hook.html | 21 +-
_modules/airflow/hooks/mssql_hook.html | 21 +-
_modules/airflow/hooks/mysql_hook.html | 33 +-
_modules/airflow/hooks/oracle_hook.html | 21 +-
_modules/airflow/hooks/pig_hook.html | 21 +-
_modules/airflow/hooks/postgres_hook.html | 21 +-
_modules/airflow/hooks/presto_hook.html | 21 +-
_modules/airflow/hooks/samba_hook.html | 21 +-
_modules/airflow/hooks/slack_hook.html | 21 +-
_modules/airflow/hooks/sqlite_hook.html | 21 +-
_modules/airflow/hooks/webhdfs_hook.html | 21 +-
_modules/airflow/hooks/zendesk_hook.html | 21 +-
_modules/airflow/macros.html | 21 +-
_modules/airflow/macros/hive.html | 21 +-
_modules/airflow/models.html | 242 +-
_modules/airflow/operators/bash_operator.html | 25 +-
_modules/airflow/operators/check_operator.html | 33 +-
_modules/airflow/operators/dagrun_operator.html | 25 +-
_modules/airflow/operators/docker_operator.html | 80 +-
.../airflow/operators/druid_check_operator.html | 25 +-
_modules/airflow/operators/dummy_operator.html | 25 +-
_modules/airflow/operators/email_operator.html | 25 +-
_modules/airflow/operators/generic_transfer.html | 25 +-
_modules/airflow/operators/hive_operator.html | 33 +-
.../airflow/operators/hive_stats_operator.html | 25 +-
_modules/airflow/operators/hive_to_druid.html | 25 +-
_modules/airflow/operators/hive_to_mysql.html | 25 +-
.../airflow/operators/hive_to_samba_operator.html | 25 +-
_modules/airflow/operators/http_operator.html | 36 +-
_modules/airflow/operators/jdbc_operator.html | 25 +-
.../airflow/operators/latest_only_operator.html | 25 +-
_modules/airflow/operators/mssql_operator.html | 25 +-
_modules/airflow/operators/mssql_to_hive.html | 25 +-
_modules/airflow/operators/mysql_operator.html | 25 +-
_modules/airflow/operators/mysql_to_hive.html | 25 +-
_modules/airflow/operators/oracle_operator.html | 25 +-
_modules/airflow/operators/pig_operator.html | 33 +-
_modules/airflow/operators/postgres_operator.html | 25 +-
.../airflow/operators/presto_check_operator.html | 21 +-
_modules/airflow/operators/presto_to_mysql.html | 25 +-
_modules/airflow/operators/python_operator.html | 33 +-
.../airflow/operators/redshift_to_s3_operator.html | 25 +-
.../operators/s3_file_transform_operator.html | 49 +-
.../airflow/operators/s3_to_hive_operator.html | 27 +-
.../airflow/operators/s3_to_redshift_operator.html | 25 +-
_modules/airflow/operators/slack_operator.html | 21 +-
_modules/airflow/operators/sqlite_operator.html | 25 +-
_modules/airflow/operators/subdag_operator.html | 25 +-
_modules/airflow/sensors/base_sensor_operator.html | 25 +-
_modules/airflow/sensors/external_task_sensor.html | 21 +-
_modules/airflow/sensors/hdfs_sensor.html | 21 +-
.../airflow/sensors/hive_partition_sensor.html | 21 +-
_modules/airflow/sensors/http_sensor.html | 21 +-
.../sensors/metastore_partition_sensor.html | 21 +-
.../sensors/named_hive_partition_sensor.html | 21 +-
_modules/airflow/sensors/s3_key_sensor.html | 21 +-
_modules/airflow/sensors/s3_prefix_sensor.html | 23 +-
_modules/airflow/sensors/sql_sensor.html | 21 +-
_modules/airflow/sensors/time_delta_sensor.html | 21 +-
_modules/airflow/sensors/time_sensor.html | 21 +-
_modules/airflow/sensors/web_hdfs_sensor.html | 21 +-
_modules/index.html | 45 +-
_sources/api.rst.txt | 17 +
_sources/cli.rst.txt | 17 +
_sources/code.rst.txt | 40 +-
_sources/concepts.rst.txt | 63 +-
_sources/faq.rst.txt | 17 +
_sources/howto/executor/use-celery.rst.txt | 19 +-
_sources/howto/executor/use-dask.rst.txt | 17 +
_sources/howto/executor/use-mesos.rst.txt | 17 +
_sources/howto/index.rst.txt | 17 +
_sources/howto/initialize-database.rst.txt | 17 +
_sources/howto/manage-connections.rst.txt | 101 +-
_sources/howto/operator.rst.txt | 493 +++
_sources/howto/run-with-systemd.rst.txt | 17 +
_sources/howto/run-with-upstart.rst.txt | 17 +
_sources/howto/secure-connections.rst.txt | 28 +-
_sources/howto/set-config.rst.txt | 37 +-
_sources/howto/use-test-config.rst.txt | 17 +
_sources/howto/write-logs.rst.txt | 86 +-
_sources/index.rst.txt | 17 +
_sources/installation.rst.txt | 84 +-
_sources/integration.rst.txt | 161 +-
_sources/kubernetes.rst.txt | 34 +-
_sources/license.rst.txt | 17 +
_sources/lineage.rst.txt | 17 +
_sources/plugins.rst.txt | 70 +-
_sources/profiling.rst.txt | 19 +
_sources/project.rst.txt | 18 +-
_sources/scheduler.rst.txt | 24 +-
_sources/security.rst.txt | 37 +-
_sources/start.rst.txt | 17 +
_sources/timezone.rst.txt | 84 +-
_sources/tutorial.rst.txt | 58 +-
_sources/ui.rst.txt | 19 +-
_static/basic.css | 11 +
_static/css/theme.css | 6 +-
_static/doctools.js | 6 +-
_static/documentation_options.js | 291 +-
_static/fonts/Inconsolata-Bold.ttf | Bin 0 -> 109948 bytes
_static/fonts/Inconsolata-Regular.ttf | Bin 0 -> 96964 bytes
_static/fonts/Inconsolata.ttf | Bin 0 -> 63184 bytes
_static/fonts/Lato-Bold.ttf | Bin 0 -> 656544 bytes
_static/fonts/Lato-Regular.ttf | Bin 0 -> 656568 bytes
_static/fonts/RobotoSlab-Bold.ttf | Bin 0 -> 170616 bytes
_static/fonts/RobotoSlab-Regular.ttf | Bin 0 -> 169064 bytes
_static/js/theme.js | 6 +-
_static/searchtools.js | 351 +--
api.html | 21 +-
cli.html | 1473 +++++----
code.html | 3218 +++++++++++++++++---
concepts.html | 59 +-
faq.html | 21 +-
genindex.html | 545 +++-
howto/executor/use-celery.html | 23 +-
howto/executor/use-dask.html | 21 +-
howto/executor/use-mesos.html | 21 +-
howto/index.html | 21 +-
howto/initialize-database.html | 21 +-
howto/manage-connections.html | 100 +-
howto/operator.html | 643 +++-
howto/run-with-systemd.html | 21 +-
howto/run-with-upstart.html | 21 +-
howto/secure-connections.html | 31 +-
howto/set-config.html | 40 +-
howto/use-test-config.html | 21 +-
howto/write-logs.html | 92 +-
index.html | 137 +-
installation.html | 124 +-
integration.html | 1798 +++++++++--
kubernetes.html | 58 +-
license.html | 21 +-
lineage.html | 21 +-
objects.inv | Bin 7505 -> 8915 bytes
plugins.html | 68 +-
profiling.html | 21 +-
project.html | 22 +-
py-modindex.html | 21 +-
scheduler.html | 27 +-
search.html | 23 +-
searchindex.js | 2 +-
security.html | 41 +-
start.html | 21 +-
timezone.html | 25 +-
tutorial.html | 62 +-
ui.html | 23 +-
292 files changed, 17934 insertions(+), 6968 deletions(-)
diff --git a/_images/airflow.gif b/_images/airflow.gif
index 1889b86..7646e37 100644
Binary files a/_images/airflow.gif and b/_images/airflow.gif differ
diff --git a/_modules/airflow/contrib/executors/mesos_executor.html b/_modules/airflow/contrib/executors/mesos_executor.html
deleted file mode 100644
index a495421..0000000
--- a/_modules/airflow/contrib/executors/mesos_executor.html
+++ /dev/null
@@ -1,548 +0,0 @@
-
-
-<!DOCTYPE html>
-<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
-<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
-<head>
- <meta charset="utf-8">
-
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
-
- <title>airflow.contrib.executors.mesos_executor — Airflow Documentation</title>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
- <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
- <link rel="index" title="Index" href="../../../../genindex.html" />
- <link rel="search" title="Search" href="../../../../search.html" />
-
-
- <script src="../../../../_static/js/modernizr.min.js"></script>
-
-</head>
-
-<body class="wy-body-for-nav">
-
-
- <div class="wy-grid-for-nav">
-
-
- <nav data-toggle="wy-nav-shift" class="wy-nav-side">
- <div class="wy-side-scroll">
- <div class="wy-side-nav-search">
-
-
-
- <a href="../../../../index.html" class="icon icon-home"> Airflow
-
-
-
- </a>
-
-
-
-
-
-
-
-<div role="search">
- <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
- <input type="text" name="q" placeholder="Search docs" />
- <input type="hidden" name="check_keywords" value="yes" />
- <input type="hidden" name="area" value="default" />
- </form>
-</div>
-
-
- </div>
-
- <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
-
-
-
-
-
-
- <ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
-</ul>
-
-
-
- </div>
- </div>
- </nav>
-
- <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
-
-
- <nav class="wy-nav-top" aria-label="top navigation">
-
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="../../../../index.html">Airflow</a>
-
- </nav>
-
-
- <div class="wy-nav-content">
-
- <div class="rst-content">
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-<div role="navigation" aria-label="breadcrumbs navigation">
-
- <ul class="wy-breadcrumbs">
-
- <li><a href="../../../../index.html">Docs</a> »</li>
-
- <li><a href="../../../index.html">Module code</a> »</li>
-
- <li>airflow.contrib.executors.mesos_executor</li>
-
-
- <li class="wy-breadcrumbs-aside">
-
- </li>
-
- </ul>
-
-
- <hr/>
-</div>
- <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
- <div itemprop="articleBody">
-
- <h1>Source code for airflow.contrib.executors.mesos_executor</h1><div class="highlight"><pre>
-<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
-<span class="c1">#</span>
-<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
-<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
-<span class="c1"># distributed with this work for additional information</span>
-<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
-<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
-<span class="c1"># "License"); you may not use this file except in compliance</span>
-<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
-<span class="c1"># software distributed under the License is distributed on an</span>
-<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
-<span class="c1"># KIND, either express or implied. See the License for the</span>
-<span class="c1"># specific language governing permissions and limitations</span>
-<span class="c1"># under the License.</span>
-
-<span class="kn">from</span> <span class="nn">future</span> <span class="k">import</span> <span class="n">standard_library</span>
-
-<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
-<span class="kn">from</span> <span class="nn">airflow.www.utils</span> <span class="k">import</span> <span class="n">LoginMixin</span>
-
-
-<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
-<span class="kn">from</span> <span class="nn">queue</span> <span class="k">import</span> <span class="n">Queue</span>
-
-<span class="kn">import</span> <span class="nn">mesos.interface</span>
-<span class="kn">from</span> <span class="nn">mesos.interface</span> <span class="k">import</span> <span class="n">mesos_pb2</span>
-<span class="kn">import</span> <span class="nn">mesos.native</span>
-
-<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">configuration</span>
-<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="k">import</span> <span class="n">BaseExecutor</span>
-<span class="kn">from</span> <span class="nn">airflow.settings</span> <span class="k">import</span> <span class="n">Session</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span>
-<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
-
-<span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
-<span class="n">DEFAULT_FRAMEWORK_NAME</span> <span class="o">=</span> <span class="s1">'Airflow'</span>
-<span class="n">FRAMEWORK_CONNID_PREFIX</span> <span class="o">=</span> <span class="s1">'mesos_framework_'</span>
-
-
-<span class="k">def</span> <span class="nf">get_framework_name</span><span class="p">():</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FRAMEWORK_NAME'</span><span class="p">):</span>
- <span class="k">return</span> <span class="n">DEFAULT_FRAMEWORK_NAME</span>
- <span class="k">return</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FRAMEWORK_NAME'</span><span class="p">)</span>
-
-
-<span class="c1"># AirflowMesosScheduler, implements Mesos Scheduler interface</span>
-<span class="c1"># To schedule airflow jobs on mesos</span>
-<span class="k">class</span> <span class="nc">AirflowMesosScheduler</span><span class="p">(</span><span class="n">mesos</span><span class="o">.</span><span class="n">interface</span><span class="o">.</span><span class="n">Scheduler</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Airflow Mesos scheduler implements mesos scheduler interface</span>
-<span class="sd"> to schedule airflow tasks on mesos.</span>
-<span class="sd"> Basically, it schedules a command like</span>
-<span class="sd"> 'airflow run <dag_id> <task_instance_id> <start_date> --local -p=<pickle>'</span>
-<span class="sd"> to run on a mesos slave.</span>
-<span class="sd"> """</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
- <span class="n">task_queue</span><span class="p">,</span>
- <span class="n">result_queue</span><span class="p">,</span>
- <span class="n">task_cpu</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
- <span class="n">task_mem</span><span class="o">=</span><span class="mi">256</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span> <span class="o">=</span> <span class="n">task_queue</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="n">result_queue</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span> <span class="o">=</span> <span class="n">task_cpu</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span> <span class="o">=</span> <span class="n">task_mem</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">task_counter</span> <span class="o">=</span> <span class="mi">0</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span> <span class="o">=</span> <span class="p">{}</span>
- <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DOCKER_IMAGE_SLAVE'</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">mesos_slave_docker_image</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
- <span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DOCKER_IMAGE_SLAVE'</span>
- <span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">registered</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">frameworkId</span><span class="p">,</span> <span class="n">masterInfo</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler registered to Mesos with framework ID </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
- <span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
-
- <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">)</span> <span class="ow">and</span> \
- <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span>
- <span class="c1"># Import here to work around a circular import error</span>
- <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">Connection</span>
-
- <span class="c1"># Update the Framework ID in the database.</span>
- <span class="n">session</span> <span class="o">=</span> <span class="n">Session</span><span class="p">()</span>
- <span class="n">conn_id</span> <span class="o">=</span> <span class="n">FRAMEWORK_CONNID_PREFIX</span> <span class="o">+</span> <span class="n">get_framework_name</span><span class="p">()</span>
- <span class="n">connection</span> <span class="o">=</span> <span class="n">Session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">Connection</span><span class="p">)</span><span class="o">.</span><span class="n">filter_by</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
- <span class="k">if</span> <span class="n">connection</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
- <span class="n">connection</span> <span class="o">=</span> <span class="n">Connection</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">,</span> <span class="n">conn_type</span><span class="o">=</span><span class="s1">'mesos_framework-id'</span><span class="p">,</span>
- <span class="n">extra</span><span class="o">=</span><span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">connection</span><span class="o">.</span><span class="n">extra</span> <span class="o">=</span> <span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span>
-
- <span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">connection</span><span class="p">)</span>
- <span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
- <span class="n">Session</span><span class="o">.</span><span class="n">remove</span><span class="p">()</span>
-
- <span class="k">def</span> <span class="nf">reregistered</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">masterInfo</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler re-registered to mesos"</span><span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">disconnected</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler disconnected from mesos"</span><span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">offerRescinded</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">offerId</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler offer </span><span class="si">%s</span><span class="s2"> rescinded"</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">offerId</span><span class="p">))</span>
-
- <span class="k">def</span> <span class="nf">frameworkMessage</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">executorId</span><span class="p">,</span> <span class="n">slaveId</span><span class="p">,</span> <span class="n">message</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler received framework message </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">executorLost</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">executorId</span><span class="p">,</span> <span class="n">slaveId</span><span class="p">,</span> <span class="n">status</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"AirflowScheduler executor </span><span class="si">%s</span><span class="s2"> lost"</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">executorId</span><span class="p">))</span>
-
- <span class="k">def</span> <span class="nf">slaveLost</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">slaveId</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"AirflowScheduler slave </span><span class="si">%s</span><span class="s2"> lost"</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">slaveId</span><span class="p">))</span>
-
- <span class="k">def</span> <span class="nf">error</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">message</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"AirflowScheduler driver aborted </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"AirflowScheduler driver aborted </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">message</span><span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">resourceOffers</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">offers</span><span class="p">):</span>
- <span class="k">for</span> <span class="n">offer</span> <span class="ow">in</span> <span class="n">offers</span><span class="p">:</span>
- <span class="n">tasks</span> <span class="o">=</span> <span class="p">[]</span>
- <span class="n">offerCpus</span> <span class="o">=</span> <span class="mi">0</span>
- <span class="n">offerMem</span> <span class="o">=</span> <span class="mi">0</span>
- <span class="k">for</span> <span class="n">resource</span> <span class="ow">in</span> <span class="n">offer</span><span class="o">.</span><span class="n">resources</span><span class="p">:</span>
- <span class="k">if</span> <span class="n">resource</span><span class="o">.</span><span class="n">name</span> <span class="o">==</span> <span class="s2">"cpus"</span><span class="p">:</span>
- <span class="n">offerCpus</span> <span class="o">+=</span> <span class="n">resource</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span>
- <span class="k">elif</span> <span class="n">resource</span><span class="o">.</span><span class="n">name</span> <span class="o">==</span> <span class="s2">"mem"</span><span class="p">:</span>
- <span class="n">offerMem</span> <span class="o">+=</span> <span class="n">resource</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span>
-
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Received offer </span><span class="si">%s</span><span class="s2"> with cpus: </span><span class="si">%s</span><span class="s2"> and mem: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
- <span class="n">offer</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">value</span><span class="p">,</span> <span class="n">offerCpus</span><span class="p">,</span> <span class="n">offerMem</span><span class="p">)</span>
-
- <span class="n">remainingCpus</span> <span class="o">=</span> <span class="n">offerCpus</span>
- <span class="n">remainingMem</span> <span class="o">=</span> <span class="n">offerMem</span>
-
- <span class="k">while</span> <span class="p">(</span><span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">empty</span><span class="p">())</span> <span class="ow">and</span> \
- <span class="n">remainingCpus</span> <span class="o">>=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span> <span class="ow">and</span> \
- <span class="n">remainingMem</span> <span class="o">>=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span><span class="p">:</span>
- <span class="n">key</span><span class="p">,</span> <span class="n">cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
- <span class="n">tid</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_counter</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">task_counter</span> <span class="o">+=</span> <span class="mi">1</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span><span class="p">[</span><span class="nb">str</span><span class="p">(</span><span class="n">tid</span><span class="p">)]</span> <span class="o">=</span> <span class="n">key</span>
-
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Launching task </span><span class="si">%d</span><span class="s2"> using offer </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">tid</span><span class="p">,</span> <span class="n">offer</span><span class="o">.</span><span class="n">id</span><span [...]
-
- <span class="n">task</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TaskInfo</span><span class="p">()</span>
- <span class="n">task</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">tid</span><span class="p">)</span>
- <span class="n">task</span><span class="o">.</span><span class="n">slave_id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">offer</span><span class="o">.</span><span class="n">slave_id</span><span class="o">.</span><span class="n">value</span>
- <span class="n">task</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="s2">"AirflowTask </span><span class="si">%d</span><span class="s2">"</span> <span class="o">%</span> <span class="n">tid</span>
-
- <span class="n">cpus</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">add</span><span class="p">()</span>
- <span class="n">cpus</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="s2">"cpus"</span>
- <span class="n">cpus</span><span class="o">.</span><span class="n">type</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">Value</span><span class="o">.</span><span class="n">SCALAR</span>
- <span class="n">cpus</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span>
-
- <span class="n">mem</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">add</span><span class="p">()</span>
- <span class="n">mem</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="s2">"mem"</span>
- <span class="n">mem</span><span class="o">.</span><span class="n">type</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">Value</span><span class="o">.</span><span class="n">SCALAR</span>
- <span class="n">mem</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span>
-
- <span class="n">command</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">CommandInfo</span><span class="p">()</span>
- <span class="n">command</span><span class="o">.</span><span class="n">shell</span> <span class="o">=</span> <span class="kc">True</span>
- <span class="n">command</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">cmd</span>
- <span class="n">task</span><span class="o">.</span><span class="n">command</span><span class="o">.</span><span class="n">MergeFrom</span><span class="p">(</span><span class="n">command</span><span class="p">)</span>
-
- <span class="c1"># If docker image for airflow is specified in config then pull that</span>
- <span class="c1"># image before running the above airflow command</span>
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mesos_slave_docker_image</span><span class="p">:</span>
- <span class="n">network</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="o">.</span><span class="n">DockerInfo</span><span class="o">.</span><span class="n">Network</span><span class="o">.</span><span class="n">Value</span><span class="p">(</span><span class="s1">'BRIDGE'</span><span class="p">)</span>
- <span class="n">docker</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="o">.</span><span class="n">DockerInfo</span><span class="p">(</span>
- <span class="n">image</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mesos_slave_docker_image</span><span class="p">,</span>
- <span class="n">force_pull_image</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">network</span><span class="o">=</span><span class="n">network</span>
- <span class="p">)</span>
- <span class="n">container</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="p">(</span>
- <span class="nb">type</span><span class="o">=</span><span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="o">.</span><span class="n">DOCKER</span><span class="p">,</span>
- <span class="n">docker</span><span class="o">=</span><span class="n">docker</span>
- <span class="p">)</span>
- <span class="n">task</span><span class="o">.</span><span class="n">container</span><span class="o">.</span><span class="n">MergeFrom</span><span class="p">(</span><span class="n">container</span><span class="p">)</span>
-
- <span class="n">tasks</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">task</span><span class="p">)</span>
-
- <span class="n">remainingCpus</span> <span class="o">-=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span>
- <span class="n">remainingMem</span> <span class="o">-=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span>
-
- <span class="n">driver</span><span class="o">.</span><span class="n">launchTasks</span><span class="p">(</span><span class="n">offer</span><span class="o">.</span><span class="n">id</span><span class="p">,</span> <span class="n">tasks</span><span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">statusUpdate</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">update</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
- <span class="s2">"Task </span><span class="si">%s</span><span class="s2"> is in state </span><span class="si">%s</span><span class="s2">, data </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
- <span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">,</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TaskState</span><span class="o">.</span><span class="n">Name</span><span class="p">(</span><span class="n">update</span><span class="o">.</span><span class="n">state</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span [...]
- <span class="p">)</span>
-
- <span class="k">try</span><span class="p">:</span>
- <span class="n">key</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span><span class="p">[</span><span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">]</span>
- <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
- <span class="c1"># The map may not contain an item if the framework re-registered</span>
- <span class="c1"># after a failover.</span>
- <span class="c1"># Discard these tasks.</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Unrecognised task key </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
- <span class="k">return</span>
-
- <span class="k">if</span> <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_FINISHED</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">))</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">task_done</span><span class="p">()</span>
-
- <span class="k">if</span> <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_LOST</span> <span class="ow">or</span> \
- <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_KILLED</span> <span class="ow">or</span> \
- <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_FAILED</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">FAILED</span><span class="p">))</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">task_done</span><span class="p">()</span>
-
-
-<div class="viewcode-block" id="MesosExecutor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor">[docs]</a><span class="k">class</span> <span class="nc">MesosExecutor</span><span class="p">(</span><span class="n">BaseExecutor</span><span class="p">,</span> <span class="n">LoginMixin</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> MesosExecutor allows distributing the execution of task</span>
-<span class="sd"> instances to multiple mesos workers.</span>
-
-<span class="sd"> Apache Mesos is a distributed systems kernel which abstracts</span>
-<span class="sd"> CPU, memory, storage, and other compute resources away from</span>
-<span class="sd"> machines (physical or virtual), enabling fault-tolerant and</span>
-<span class="sd"> elastic distributed systems to easily be built and run effectively.</span>
-<span class="sd"> See http://mesos.apache.org/</span>
-<span class="sd"> """</span>
-<div class="viewcode-block" id="MesosExecutor.start"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor.start">[docs]</a> <span class="k">def</span> <span class="nf">start</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span> <span class="o">=</span> <span class="n">Queue</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="n">Queue</span><span class="p">()</span>
- <span class="n">framework</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">FrameworkInfo</span><span class="p">()</span>
- <span class="n">framework</span><span class="o">.</span><span class="n">user</span> <span class="o">=</span> <span class="s1">''</span>
-
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'MASTER'</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Expecting mesos master URL for mesos executor"</span><span class="p">)</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"mesos.master not provided for mesos executor"</span><span class="p">)</span>
-
- <span class="n">master</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'MASTER'</span><span class="p">)</span>
-
- <span class="n">framework</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">get_framework_name</span><span class="p">()</span>
-
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_CPU'</span><span class="p">):</span>
- <span class="n">task_cpu</span> <span class="o">=</span> <span class="mi">1</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">task_cpu</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_CPU'</span><span class="p">)</span>
-
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_MEMORY'</span><span class="p">):</span>
- <span class="n">task_memory</span> <span class="o">=</span> <span class="mi">256</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">task_memory</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_MEMORY'</span><span class="p">)</span>
-
- <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">):</span>
- <span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="kc">True</span>
-
- <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span>
- <span class="c1"># Import here to work around a circular import error</span>
- <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">Connection</span>
-
- <span class="c1"># Query the database to get the ID of the Mesos Framework, if available.</span>
- <span class="n">conn_id</span> <span class="o">=</span> <span class="n">FRAMEWORK_CONNID_PREFIX</span> <span class="o">+</span> <span class="n">framework</span><span class="o">.</span><span class="n">name</span>
- <span class="n">session</span> <span class="o">=</span> <span class="n">Session</span><span class="p">()</span>
- <span class="n">connection</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">Connection</span><span class="p">)</span><span class="o">.</span><span class="n">filter_by</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p [...]
- <span class="k">if</span> <span class="n">connection</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
- <span class="c1"># Set the Framework ID to let the scheduler reconnect</span>
- <span class="c1"># with running tasks.</span>
- <span class="n">framework</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">connection</span><span class="o">.</span><span class="n">extra</span>
-
- <span class="n">framework</span><span class="o">.</span><span class="n">failover_timeout</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span>
- <span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span>
- <span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="kc">False</span>
-
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
- <span class="s1">'MesosFramework master : </span><span class="si">%s</span><span class="s1">, name : </span><span class="si">%s</span><span class="s1">, cpu : </span><span class="si">%s</span><span class="s1">, mem : </span><span class="si">%s</span><span class="s1">, checkpoint : </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
- <span class="n">master</span><span class="p">,</span> <span class="n">framework</span><span class="o">.</span><span class="n">name</span><span class="p">,</span>
- <span class="nb">str</span><span class="p">(</span><span class="n">task_cpu</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">task_memory</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span><span class="p">)</span>
- <span class="p">)</span>
-
- <span class="n">implicit_acknowledgements</span> <span class="o">=</span> <span class="mi">1</span>
-
- <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'AUTHENTICATE'</span><span class="p">):</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_PRINCIPAL'</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Expecting authentication principal in the environment"</span><span class="p">)</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
- <span class="s2">"mesos.default_principal not provided in authenticated mode"</span><span class="p">)</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_SECRET'</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Expecting authentication secret in the environment"</span><span class="p">)</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
- <span class="s2">"mesos.default_secret not provided in authenticated mode"</span><span class="p">)</span>
-
- <span class="n">credential</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">Credential</span><span class="p">()</span>
- <span class="n">credential</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_PRINCIPAL'</span><span class="p">)</span>
- <span class="n">credential</span><span class="o">.</span><span class="n">secret</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_SECRET'</span><span class="p">)</span>
-
- <span class="n">framework</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="n">credential</span><span class="o">.</span><span class="n">principal</span>
-
- <span class="n">driver</span> <span class="o">=</span> <span class="n">mesos</span><span class="o">.</span><span class="n">native</span><span class="o">.</span><span class="n">MesosSchedulerDriver</span><span class="p">(</span>
- <span class="n">AirflowMesosScheduler</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="p">,</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="p">,</span>
- <span class="n">task_cpu</span><span class="p">,</span>
- <span class="n">task_memory</span><span class="p">),</span>
- <span class="n">framework</span><span class="p">,</span>
- <span class="n">master</span><span class="p">,</span>
- <span class="n">implicit_acknowledgements</span><span class="p">,</span>
- <span class="n">credential</span><span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">framework</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="s1">'Airflow'</span>
- <span class="n">driver</span> <span class="o">=</span> <span class="n">mesos</span><span class="o">.</span><span class="n">native</span><span class="o">.</span><span class="n">MesosSchedulerDriver</span><span class="p">(</span>
- <span class="n">AirflowMesosScheduler</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="p">,</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="p">,</span>
- <span class="n">task_cpu</span><span class="p">,</span>
- <span class="n">task_memory</span><span class="p">),</span>
- <span class="n">framework</span><span class="p">,</span>
- <span class="n">master</span><span class="p">,</span>
- <span class="n">implicit_acknowledgements</span><span class="p">)</span>
-
- <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span> <span class="o">=</span> <span class="n">driver</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span><span class="o">.</span><span class="n">start</span><span class="p">()</span></div>
-
-<div class="viewcode-block" id="MesosExecutor.execute_async"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor.execute_async">[docs]</a> <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</ [...]
- <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">))</span></div>
-
-<div class="viewcode-block" id="MesosExecutor.sync"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor.sync">[docs]</a> <span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">empty</span><span class="p">():</span>
- <span class="n">results</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">change_state</span><span class="p">(</span><span class="o">*</span><span class="n">results</span><span class="p">)</span></div>
-
-<div class="viewcode-block" id="MesosExecutor.end"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor.end">[docs]</a> <span class="k">def</span> <span class="nf">end</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span><span class="o">.</span><span class="n">stop</span><span class="p">()</span></div></div>
-</pre></div>
-
- </div>
-
- </div>
- <footer>
-
-
- <hr/>
-
- <div role="contentinfo">
- <p>
-
- </p>
- </div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
-
-</footer>
-
- </div>
- </div>
-
- </section>
-
- </div>
-
-
-
-
-
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
-
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
- <script type="text/javascript">
- jQuery(function () {
- SphinxRtdTheme.Navigation.enable(true);
- });
- </script>
-
-</body>
-</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html b/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html
index b8fea44..4feb169 100644
--- a/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html
+++ b/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html
@@ -258,20 +258,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/operators/vertica_operator.html b/_modules/airflow/contrib/hooks/aws_firehose_hook.html
similarity index 62%
copy from _modules/airflow/contrib/operators/vertica_operator.html
copy to _modules/airflow/contrib/hooks/aws_firehose_hook.html
index 49b6fe6..c78e6db 100644
--- a/_modules/airflow/contrib/operators/vertica_operator.html
+++ b/_modules/airflow/contrib/hooks/aws_firehose_hook.html
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <title>airflow.contrib.operators.vertica_operator — Airflow Documentation</title>
+ <title>airflow.contrib.hooks.aws_firehose_hook — Airflow Documentation</title>
@@ -143,7 +143,7 @@
<li><a href="../../../index.html">Module code</a> »</li>
- <li>airflow.contrib.operators.vertica_operator</li>
+ <li>airflow.contrib.hooks.aws_firehose_hook</li>
<li class="wy-breadcrumbs-aside">
@@ -158,9 +158,8 @@
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
- <h1>Source code for airflow.contrib.operators.vertica_operator</h1><div class="highlight"><pre>
-<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
-<span class="c1">#</span>
+ <h1>Source code for airflow.contrib.hooks.aws_firehose_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1">#</span>
<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
<span class="c1"># distributed with this work for additional information</span>
@@ -177,37 +176,45 @@
<span class="c1"># KIND, either express or implied. See the License for the</span>
<span class="c1"># specific language governing permissions and limitations</span>
<span class="c1"># under the License.</span>
-<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.vertica_hook</span> <span class="k">import</span> <span class="n">VerticaHook</span>
-<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.aws_hook</span> <span class="k">import</span> <span class="n">AwsHook</span>
-<div class="viewcode-block" id="VerticaOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.vertica_operator.VerticaOperator">[docs]</a><span class="k">class</span> <span class="nc">VerticaOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+<div class="viewcode-block" id="AwsFirehoseHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_firehose_hook.AwsFirehoseHook">[docs]</a><span class="k">class</span> <span class="nc">AwsFirehoseHook</span><span class="p">(</span><span class="n">AwsHook</span><span class="p">):</span>
<span class="sd">"""</span>
-<span class="sd"> Executes sql code in a specific Vertica database</span>
-
-<span class="sd"> :param vertica_conn_id: reference to a specific Vertica database</span>
-<span class="sd"> :type vertica_conn_id: string</span>
-<span class="sd"> :param sql: the sql code to be executed. (templated)</span>
-<span class="sd"> :type sql: Can receive a str representing a sql statement,</span>
-<span class="sd"> a list of str (sql statements), or reference to a template file.</span>
-<span class="sd"> Template reference are recognized by str ending in '.sql'</span>
+<span class="sd"> Interact with AWS Kinesis Firehose.</span>
+<span class="sd"> :param delivery_stream: Name of the delivery stream</span>
+<span class="sd"> :type delivery_stream: str</span>
+<span class="sd"> :param region_name: AWS region name (example: us-east-1)</span>
+<span class="sd"> :type region_name: str</span>
<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'sql'</span><span class="p">,)</span>
- <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,)</span>
- <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#b4e0ff'</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">delivery_stream</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delivery_stream</span> <span class="o">=</span> <span class="n">delivery_stream</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">region_name</span> <span class="o">=</span> <span class="n">region_name</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">AwsFirehoseHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+<div class="viewcode-block" id="AwsFirehoseHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_firehose_hook.AwsFirehoseHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns AwsHook connection object.</span>
+<span class="sd"> """</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_client_type</span><span class="p">(</span><span class="s1">'firehose'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">region_name</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span></div>
+
+<div class="viewcode-block" id="AwsFirehoseHook.put_records"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_firehose_hook.AwsFirehoseHook.put_records">[docs]</a> <span class="k">def</span> <span class="nf">put_records</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">records</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Write batch records to Kinesis Firehose</span>
+<span class="sd"> """</span>
- <span class="nd">@apply_defaults</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">vertica_conn_id</span><span class="o">=</span><span class="s1">'vertica_default'</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">VerticaOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">vertica_conn_id</span> <span class="o">=</span> <span class="n">vertica_conn_id</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span>
+ <span class="n">firehose_conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
- <span class="n">hook</span> <span class="o">=</span> <span class="n">VerticaHook</span><span class="p">(</span><span class="n">vertica_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">vertica_conn_id</span><span class="p">)</span>
- <span class="n">hook</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span></div>
+ <span class="n">response</span> <span class="o">=</span> <span class="n">firehose_conn</span><span class="o">.</span><span class="n">put_record_batch</span><span class="p">(</span>
+ <span class="n">DeliveryStreamName</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delivery_stream</span><span class="p">,</span>
+ <span class="n">Records</span><span class="o">=</span><span class="n">records</span>
+ <span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">response</span></div></div>
</pre></div>
</div>
@@ -238,20 +245,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/aws_hook.html b/_modules/airflow/contrib/hooks/aws_hook.html
index f572e90..fb81ebf 100644
--- a/_modules/airflow/contrib/hooks/aws_hook.html
+++ b/_modules/airflow/contrib/hooks/aws_hook.html
@@ -178,7 +178,6 @@
<span class="c1"># specific language governing permissions and limitations</span>
<span class="c1"># under the License.</span>
-
<span class="kn">import</span> <span class="nn">boto3</span>
<span class="kn">import</span> <span class="nn">configparser</span>
<span class="kn">import</span> <span class="nn">logging</span>
@@ -233,7 +232,7 @@
<span class="k">try</span><span class="p">:</span>
<span class="n">access_key</span> <span class="o">=</span> <span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">cred_section</span><span class="p">,</span> <span class="n">key_id_option</span><span class="p">)</span>
<span class="n">secret_key</span> <span class="o">=</span> <span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">cred_section</span><span class="p">,</span> <span class="n">secret_key_option</span><span class="p">)</span>
- <span class="k">except</span><span class="p">:</span>
+ <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
<span class="n">logging</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Option Error in parsing s3 config file"</span><span class="p">)</span>
<span class="k">raise</span>
<span class="k">return</span> <span class="n">access_key</span><span class="p">,</span> <span class="n">secret_key</span>
@@ -257,32 +256,36 @@
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">connection_object</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span>
+ <span class="n">extra_config</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span>
<span class="k">if</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">login</span><span class="p">:</span>
<span class="n">aws_access_key_id</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">login</span>
<span class="n">aws_secret_access_key</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">password</span>
- <span class="k">elif</span> <span class="s1">'aws_secret_access_key'</span> <span class="ow">in</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">:</span>
- <span class="n">aws_access_key_id</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span>
+ <span class="k">elif</span> <span class="s1">'aws_secret_access_key'</span> <span class="ow">in</span> <span class="n">extra_config</span><span class="p">:</span>
+ <span class="n">aws_access_key_id</span> <span class="o">=</span> <span class="n">extra_config</span><span class="p">[</span>
<span class="s1">'aws_access_key_id'</span><span class="p">]</span>
- <span class="n">aws_secret_access_key</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span>
+ <span class="n">aws_secret_access_key</span> <span class="o">=</span> <span class="n">extra_config</span><span class="p">[</span>
<span class="s1">'aws_secret_access_key'</span><span class="p">]</span>
- <span class="k">elif</span> <span class="s1">'s3_config_file'</span> <span class="ow">in</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">:</span>
+ <span class="k">elif</span> <span class="s1">'s3_config_file'</span> <span class="ow">in</span> <span class="n">extra_config</span><span class="p">:</span>
<span class="n">aws_access_key_id</span><span class="p">,</span> <span class="n">aws_secret_access_key</span> <span class="o">=</span> \
<span class="n">_parse_s3_config</span><span class="p">(</span>
- <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s1">'s3_config_file'</span><span class="p">],</span>
- <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'s3_config_format'</span><span class="p">))</span>
+ <span class="n">extra_config</span><span class="p">[</span><span class="s1">'s3_config_file'</span><span class="p">],</span>
+ <span class="n">extra_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'s3_config_format'</span><span class="p">),</span>
+ <span class="n">extra_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'profile'</span><span class="p">))</span>
<span class="k">if</span> <span class="n">region_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
- <span class="n">region_name</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'region_name'</span><span class="p">)</span>
+ <span class="n">region_name</span> <span class="o">=</span> <span class="n">extra_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'region_name'</span><span class="p">)</span>
- <span class="n">role_arn</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'role_arn'</span><span class="p">)</span>
- <span class="n">aws_account_id</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'aws_account_id'</span><span class="p">)</span>
- <span class="n">aws_iam_role</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'aws_iam_role'</span><span class="p">)</span>
+ <span class="n">role_arn</span> <span class="o">=</span> <span class="n">extra_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'role_arn'</span><span class="p">)</span>
+ <span class="n">external_id</span> <span class="o">=</span> <span class="n">extra_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'external_id'</span><span class="p">)</span>
+ <span class="n">aws_account_id</span> <span class="o">=</span> <span class="n">extra_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'aws_account_id'</span><span class="p">)</span>
+ <span class="n">aws_iam_role</span> <span class="o">=</span> <span class="n">extra_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'aws_iam_role'</span><span class="p">)</span>
<span class="k">if</span> <span class="n">role_arn</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">aws_account_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> \
<span class="n">aws_iam_role</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
- <span class="n">role_arn</span> <span class="o">=</span> <span class="s2">"arn:aws:iam::"</span> <span class="o">+</span> <span class="n">aws_account_id</span> <span class="o">+</span> <span class="s2">":role/"</span> <span class="o">+</span> <span class="n">aws_iam_role</span>
+ <span class="n">role_arn</span> <span class="o">=</span> <span class="s2">"arn:aws:iam::</span><span class="si">{}</span><span class="s2">:role/</span><span class="si">{}</span><span class="s2">"</span> \
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">aws_account_id</span><span class="p">,</span> <span class="n">aws_iam_role</span><span class="p">)</span>
<span class="k">if</span> <span class="n">role_arn</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">sts_session</span> <span class="o">=</span> <span class="n">boto3</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">Session</span><span class="p">(</span>
@@ -291,14 +294,23 @@
<span class="n">region_name</span><span class="o">=</span><span class="n">region_name</span><span class="p">)</span>
<span class="n">sts_client</span> <span class="o">=</span> <span class="n">sts_session</span><span class="o">.</span><span class="n">client</span><span class="p">(</span><span class="s1">'sts'</span><span class="p">)</span>
- <span class="n">sts_response</span> <span class="o">=</span> <span class="n">sts_client</span><span class="o">.</span><span class="n">assume_role</span><span class="p">(</span>
- <span class="n">RoleArn</span><span class="o">=</span><span class="n">role_arn</span><span class="p">,</span>
- <span class="n">RoleSessionName</span><span class="o">=</span><span class="s1">'Airflow_'</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span>
- <span class="n">aws_access_key_id</span> <span class="o">=</span> <span class="n">sts_response</span><span class="p">[</span><span class="s1">'Credentials'</span><span class="p">][</span><span class="s1">'AccessKeyId'</span><span class="p">]</span>
- <span class="n">aws_secret_access_key</span> <span class="o">=</span> <span class="n">sts_response</span><span class="p">[</span><span class="s1">'Credentials'</span><span class="p">][</span><span class="s1">'SecretAccessKey'</span><span class="p">]</span>
- <span class="n">aws_session_token</span> <span class="o">=</span> <span class="n">sts_response</span><span class="p">[</span><span class="s1">'Credentials'</span><span class="p">][</span><span class="s1">'SessionToken'</span><span class="p">]</span>
- <span class="n">endpoint_url</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'host'</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">external_id</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">sts_response</span> <span class="o">=</span> <span class="n">sts_client</span><span class="o">.</span><span class="n">assume_role</span><span class="p">(</span>
+ <span class="n">RoleArn</span><span class="o">=</span><span class="n">role_arn</span><span class="p">,</span>
+ <span class="n">RoleSessionName</span><span class="o">=</span><span class="s1">'Airflow_'</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">sts_response</span> <span class="o">=</span> <span class="n">sts_client</span><span class="o">.</span><span class="n">assume_role</span><span class="p">(</span>
+ <span class="n">RoleArn</span><span class="o">=</span><span class="n">role_arn</span><span class="p">,</span>
+ <span class="n">RoleSessionName</span><span class="o">=</span><span class="s1">'Airflow_'</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">,</span>
+ <span class="n">ExternalId</span><span class="o">=</span><span class="n">external_id</span><span class="p">)</span>
+
+ <span class="n">credentials</span> <span class="o">=</span> <span class="n">sts_response</span><span class="p">[</span><span class="s1">'Credentials'</span><span class="p">]</span>
+ <span class="n">aws_access_key_id</span> <span class="o">=</span> <span class="n">credentials</span><span class="p">[</span><span class="s1">'AccessKeyId'</span><span class="p">]</span>
+ <span class="n">aws_secret_access_key</span> <span class="o">=</span> <span class="n">credentials</span><span class="p">[</span><span class="s1">'SecretAccessKey'</span><span class="p">]</span>
+ <span class="n">aws_session_token</span> <span class="o">=</span> <span class="n">credentials</span><span class="p">[</span><span class="s1">'SessionToken'</span><span class="p">]</span>
+
+ <span class="n">endpoint_url</span> <span class="o">=</span> <span class="n">extra_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'host'</span><span class="p">)</span>
<span class="k">except</span> <span class="n">AirflowException</span><span class="p">:</span>
<span class="c1"># No connection found: fallback on boto3 credential strategy</span>
@@ -311,15 +323,17 @@
<span class="n">aws_session_token</span><span class="o">=</span><span class="n">aws_session_token</span><span class="p">,</span>
<span class="n">region_name</span><span class="o">=</span><span class="n">region_name</span><span class="p">),</span> <span class="n">endpoint_url</span>
- <span class="k">def</span> <span class="nf">get_client_type</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">client_type</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">get_client_type</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">client_type</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="n">session</span><span class="p">,</span> <span class="n">endpoint_url</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_credentials</span><span class="p">(</span><span class="n">region_name</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">client</span><span class="p">(</span><span class="n">client_type</span><span class="p">,</span> <span class="n">endpoint_url</span><span class="o">=</span><span class="n">endpoint_url</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">client</span><span class="p">(</span><span class="n">client_type</span><span class="p">,</span> <span class="n">endpoint_url</span><span class="o">=</span><span class="n">endpoint_url</span><span class="p">,</span>
+ <span class="n">config</span><span class="o">=</span><span class="n">config</span><span class="p">)</span>
- <span class="k">def</span> <span class="nf">get_resource_type</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resource_type</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">get_resource_type</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resource_type</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="n">session</span><span class="p">,</span> <span class="n">endpoint_url</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_credentials</span><span class="p">(</span><span class="n">region_name</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">resource</span><span class="p">(</span><span class="n">resource_type</span><span class="p">,</span> <span class="n">endpoint_url</span><span class="o">=</span><span class="n">endpoint_url</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">resource</span><span class="p">(</span><span class="n">resource_type</span><span class="p">,</span> <span class="n">endpoint_url</span><span class="o">=</span><span class="n">endpoint_url</span><span class="p">,</span>
+ <span class="n">config</span><span class="o">=</span><span class="n">config</span><span class="p">)</span>
<div class="viewcode-block" id="AwsHook.get_session"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_hook.AwsHook.get_session">[docs]</a> <span class="k">def</span> <span class="nf">get_session</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""Get the underlying boto3.session."""</span>
@@ -329,13 +343,26 @@
<div class="viewcode-block" id="AwsHook.get_credentials"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_hook.AwsHook.get_credentials">[docs]</a> <span class="k">def</span> <span class="nf">get_credentials</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""Get the underlying `botocore.Credentials` object.</span>
-<span class="sd"> This contains the attributes: access_key, secret_key and token.</span>
+<span class="sd"> This contains the following authentication attributes: access_key, secret_key and token.</span>
<span class="sd"> """</span>
<span class="n">session</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_credentials</span><span class="p">(</span><span class="n">region_name</span><span class="p">)</span>
- <span class="c1"># Credentials are refreshable, so accessing your access key / secret key</span>
- <span class="c1"># separately can lead to a race condition.</span>
+ <span class="c1"># Credentials are refreshable, so accessing your access key and</span>
+ <span class="c1"># secret key separately can lead to a race condition.</span>
<span class="c1"># See https://stackoverflow.com/a/36291428/8283373</span>
- <span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">get_credentials</span><span class="p">()</span><span class="o">.</span><span class="n">get_frozen_credentials</span><span class="p">()</span></div></div>
+ <span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">get_credentials</span><span class="p">()</span><span class="o">.</span><span class="n">get_frozen_credentials</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="AwsHook.expand_role"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_hook.AwsHook.expand_role">[docs]</a> <span class="k">def</span> <span class="nf">expand_role</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">role</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> If the IAM role is a role name, get the Amazon Resource Name (ARN) for the role.</span>
+<span class="sd"> If IAM role is already an IAM role ARN, no change is made.</span>
+
+<span class="sd"> :param role: IAM role name or ARN</span>
+<span class="sd"> :return: IAM role ARN</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="s1">'/'</span> <span class="ow">in</span> <span class="n">role</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">role</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_client_type</span><span class="p">(</span><span class="s1">'iam'</span><span class="p">)</span><span class="o">.</span><span class="n">get_role</span><span class="p">(</span><span class="n">RoleName</span><span class="o">=</span><span class="n">role</span><span class="p">)[</span><span class="s1">'Role'</span><span class="p">][</span><span class="s1">'Arn& [...]
</pre></div>
</div>
@@ -366,20 +393,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/aws_lambda_hook.html b/_modules/airflow/contrib/hooks/aws_lambda_hook.html
index 131de2a..9737155 100644
--- a/_modules/airflow/contrib/hooks/aws_lambda_hook.html
+++ b/_modules/airflow/contrib/hooks/aws_lambda_hook.html
@@ -257,20 +257,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/azure_data_lake_hook.html b/_modules/airflow/contrib/hooks/azure_data_lake_hook.html
index 89e24a5..5708fd2 100644
--- a/_modules/airflow/contrib/hooks/azure_data_lake_hook.html
+++ b/_modules/airflow/contrib/hooks/azure_data_lake_hook.html
@@ -330,20 +330,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/azure_fileshare_hook.html b/_modules/airflow/contrib/hooks/azure_fileshare_hook.html
index a34fc55..e2b536a 100644
--- a/_modules/airflow/contrib/hooks/azure_fileshare_hook.html
+++ b/_modules/airflow/contrib/hooks/azure_fileshare_hook.html
@@ -401,20 +401,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/bigquery_hook.html b/_modules/airflow/contrib/hooks/bigquery_hook.html
index b02ae65..4416c41 100644
--- a/_modules/airflow/contrib/hooks/bigquery_hook.html
+++ b/_modules/airflow/contrib/hooks/bigquery_hook.html
@@ -367,7 +367,8 @@
<span class="n">dataset_id</span><span class="p">,</span>
<span class="n">table_id</span><span class="p">,</span>
<span class="n">schema_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">time_partitioning</span><span class="o">=</span><span class="p">{}</span>
+ <span class="n">time_partitioning</span><span class="o">=</span><span class="p">{},</span>
+ <span class="n">labels</span><span class="o">=</span><span class="kc">None</span>
<span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Creates a new, empty table in the dataset.</span>
@@ -380,6 +381,8 @@
<span class="sd"> :type table_id: str</span>
<span class="sd"> :param schema_fields: If set, the schema field list as defined here:</span>
<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load.schema</span>
+<span class="sd"> :param labels: a dictionary containing labels for the table, passed to BigQuery</span>
+<span class="sd"> :type labels: dict</span>
<span class="sd"> **Example**: ::</span>
@@ -410,6 +413,9 @@
<span class="k">if</span> <span class="n">time_partitioning</span><span class="p">:</span>
<span class="n">table_resource</span><span class="p">[</span><span class="s1">'timePartitioning'</span><span class="p">]</span> <span class="o">=</span> <span class="n">time_partitioning</span>
+ <span class="k">if</span> <span class="n">labels</span><span class="p">:</span>
+ <span class="n">table_resource</span><span class="p">[</span><span class="s1">'labels'</span><span class="p">]</span> <span class="o">=</span> <span class="n">labels</span>
+
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Creating Table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
<span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
@@ -441,7 +447,8 @@
<span class="n">quote_character</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">allow_quoted_newlines</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">allow_jagged_rows</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">src_fmt_configs</span><span class="o">=</span><span class="p">{}</span>
+ <span class="n">src_fmt_configs</span><span class="o">=</span><span class="p">{},</span>
+ <span class="n">labels</span><span class="o">=</span><span class="kc">None</span>
<span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Creates a new external table in the dataset with the data in Google</span>
@@ -502,6 +509,8 @@
<span class="sd"> :type allow_jagged_rows: bool</span>
<span class="sd"> :param src_fmt_configs: configure optional fields specific to the source format</span>
<span class="sd"> :type src_fmt_configs: dict</span>
+<span class="sd"> :param labels: a dictionary containing labels for the table, passed to BigQuery</span>
+<span class="sd"> :type labels: dict</span>
<span class="sd"> """</span>
<span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">external_table_id</span> <span class="o">=</span> \
@@ -600,6 +609,9 @@
<span class="n">table_resource</span><span class="p">[</span><span class="s1">'externalDataConfiguration'</span><span class="p">][</span><span class="n">src_fmt_to_param_mapping</span><span class="p">[</span>
<span class="n">source_format</span><span class="p">]]</span> <span class="o">=</span> <span class="n">src_fmt_configs</span>
+ <span class="k">if</span> <span class="n">labels</span><span class="p">:</span>
+ <span class="n">table_resource</span><span class="p">[</span><span class="s1">'labels'</span><span class="p">]</span> <span class="o">=</span> <span class="n">labels</span>
+
<span class="k">try</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span>
<span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
@@ -621,13 +633,14 @@
<span class="n">destination_dataset_table</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">write_disposition</span><span class="o">=</span><span class="s1">'WRITE_EMPTY'</span><span class="p">,</span>
<span class="n">allow_large_results</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">flatten_results</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">flatten_results</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">udf_config</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">use_legacy_sql</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">maximum_billing_tier</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">maximum_bytes_billed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">create_disposition</span><span class="o">=</span><span class="s1">'CREATE_IF_NEEDED'</span><span class="p">,</span>
<span class="n">query_params</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">schema_update_options</span><span class="o">=</span><span class="p">(),</span>
<span class="n">priority</span><span class="o">=</span><span class="s1">'INTERACTIVE'</span><span class="p">,</span>
<span class="n">time_partitioning</span><span class="o">=</span><span class="p">{}):</span>
@@ -677,6 +690,9 @@
<span class="sd"> :param query_params a dictionary containing query parameter types and</span>
<span class="sd"> values, passed to BigQuery</span>
<span class="sd"> :type query_params: dict</span>
+<span class="sd"> :param labels a dictionary containing labels for the job/query,</span>
+<span class="sd"> passed to BigQuery</span>
+<span class="sd"> :type labels: dict</span>
<span class="sd"> :param schema_update_options: Allows the schema of the desitination</span>
<span class="sd"> table to be updated as a side effect of the query job.</span>
<span class="sd"> :type schema_update_options: tuple</span>
@@ -685,9 +701,7 @@
<span class="sd"> The default value is INTERACTIVE.</span>
<span class="sd"> :type priority: string</span>
<span class="sd"> :param time_partitioning: configure optional time partitioning fields i.e.</span>
-<span class="sd"> partition by field, type and</span>
-<span class="sd"> expiration as per API specifications. Note that 'field' is not available in</span>
-<span class="sd"> conjunction with dataset.table$partition.</span>
+<span class="sd"> partition by field, type and expiration as per API specifications.</span>
<span class="sd"> :type time_partitioning: dict</span>
<span class="sd"> """</span>
@@ -737,9 +751,11 @@
<span class="p">}</span>
<span class="k">if</span> <span class="n">destination_dataset_table</span><span class="p">:</span>
- <span class="k">assert</span> <span class="s1">'.'</span> <span class="ow">in</span> <span class="n">destination_dataset_table</span><span class="p">,</span> <span class="p">(</span>
- <span class="s1">'Expected destination_dataset_table in the format of '</span>
- <span class="s1">'<dataset>.<table>. Got: </span><span class="si">{}</span><span class="s1">'</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">destination_dataset_table</span><span class="p">)</span>
+ <span class="k">if</span> <span class="s1">'.'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">destination_dataset_table</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+ <span class="s1">'Expected destination_dataset_table name in the format of '</span>
+ <span class="s1">'<dataset>.<table>. Got: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">destination_dataset_table</span><span class="p">))</span>
<span class="n">destination_project</span><span class="p">,</span> <span class="n">destination_dataset</span><span class="p">,</span> <span class="n">destination_table</span> <span class="o">=</span> \
<span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">destination_dataset_table</span><span class="p">,</span>
<span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">)</span>
@@ -755,7 +771,9 @@
<span class="p">}</span>
<span class="p">})</span>
<span class="k">if</span> <span class="n">udf_config</span><span class="p">:</span>
- <span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">udf_config</span><span class="p">,</span> <span class="nb">list</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">udf_config</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">"udf_config argument must have a type 'list'"</span>
+ <span class="s2">" not </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">udf_config</span><span class="p">)))</span>
<span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">({</span>
<span class="s1">'userDefinedFunctionResources'</span><span class="p">:</span> <span class="n">udf_config</span>
<span class="p">})</span>
@@ -767,6 +785,9 @@
<span class="k">else</span><span class="p">:</span>
<span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">][</span><span class="s1">'queryParameters'</span><span class="p">]</span> <span class="o">=</span> <span class="n">query_params</span>
+ <span class="k">if</span> <span class="n">labels</span><span class="p">:</span>
+ <span class="n">configuration</span><span class="p">[</span><span class="s1">'labels'</span><span class="p">]</span> <span class="o">=</span> <span class="n">labels</span>
+
<span class="n">time_partitioning</span> <span class="o">=</span> <span class="n">_cleanse_time_partitioning</span><span class="p">(</span>
<span class="n">destination_dataset_table</span><span class="p">,</span>
<span class="n">time_partitioning</span>
@@ -797,7 +818,8 @@
<span class="n">compression</span><span class="o">=</span><span class="s1">'NONE'</span><span class="p">,</span>
<span class="n">export_format</span><span class="o">=</span><span class="s1">'CSV'</span><span class="p">,</span>
<span class="n">field_delimiter</span><span class="o">=</span><span class="s1">','</span><span class="p">,</span>
- <span class="n">print_header</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
+ <span class="n">print_header</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+ <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Executes a BigQuery extract command to copy data from BigQuery to</span>
<span class="sd"> Google Cloud Storage. See here:</span>
@@ -822,6 +844,9 @@
<span class="sd"> :type field_delimiter: string</span>
<span class="sd"> :param print_header: Whether to print a header for a CSV file extract.</span>
<span class="sd"> :type print_header: boolean</span>
+<span class="sd"> :param labels: a dictionary containing labels for the job/query,</span>
+<span class="sd"> passed to BigQuery</span>
+<span class="sd"> :type labels: dict</span>
<span class="sd"> """</span>
<span class="n">source_project</span><span class="p">,</span> <span class="n">source_dataset</span><span class="p">,</span> <span class="n">source_table</span> <span class="o">=</span> \
@@ -842,6 +867,9 @@
<span class="p">}</span>
<span class="p">}</span>
+ <span class="k">if</span> <span class="n">labels</span><span class="p">:</span>
+ <span class="n">configuration</span><span class="p">[</span><span class="s1">'labels'</span><span class="p">]</span> <span class="o">=</span> <span class="n">labels</span>
+
<span class="k">if</span> <span class="n">export_format</span> <span class="o">==</span> <span class="s1">'CSV'</span><span class="p">:</span>
<span class="c1"># Only set fieldDelimiter and printHeader fields if using CSV.</span>
<span class="c1"># Google does not like it if you set these fields for other export</span>
@@ -855,7 +883,8 @@
<span class="n">source_project_dataset_tables</span><span class="p">,</span>
<span class="n">destination_project_dataset_table</span><span class="p">,</span>
<span class="n">write_disposition</span><span class="o">=</span><span class="s1">'WRITE_EMPTY'</span><span class="p">,</span>
- <span class="n">create_disposition</span><span class="o">=</span><span class="s1">'CREATE_IF_NEEDED'</span><span class="p">):</span>
+ <span class="n">create_disposition</span><span class="o">=</span><span class="s1">'CREATE_IF_NEEDED'</span><span class="p">,</span>
+ <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Executes a BigQuery copy command to copy data from one BigQuery table</span>
<span class="sd"> to another. See here:</span>
@@ -878,6 +907,9 @@
<span class="sd"> :type write_disposition: string</span>
<span class="sd"> :param create_disposition: The create disposition if the table doesn't exist.</span>
<span class="sd"> :type create_disposition: string</span>
+<span class="sd"> :param labels a dictionary containing labels for the job/query,</span>
+<span class="sd"> passed to BigQuery</span>
+<span class="sd"> :type labels: dict</span>
<span class="sd"> """</span>
<span class="n">source_project_dataset_tables</span> <span class="o">=</span> <span class="p">([</span>
<span class="n">source_project_dataset_tables</span>
@@ -915,6 +947,9 @@
<span class="p">}</span>
<span class="p">}</span>
+ <span class="k">if</span> <span class="n">labels</span><span class="p">:</span>
+ <span class="n">configuration</span><span class="p">[</span><span class="s1">'labels'</span><span class="p">]</span> <span class="o">=</span> <span class="n">labels</span>
+
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_with_configuration</span><span class="p">(</span><span class="n">configuration</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">run_load</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
@@ -993,9 +1028,7 @@
<span class="sd"> :param src_fmt_configs: configure optional fields specific to the source format</span>
<span class="sd"> :type src_fmt_configs: dict</span>
<span class="sd"> :param time_partitioning: configure optional time partitioning fields i.e.</span>
-<span class="sd"> partition by field, type and</span>
-<span class="sd"> expiration as per API specifications. Note that 'field' is not available in</span>
-<span class="sd"> conjunction with dataset.table$partition.</span>
+<span class="sd"> partition by field, type and expiration as per API specifications.</span>
<span class="sd"> :type time_partitioning: dict</span>
<span class="sd"> """</span>
@@ -1281,10 +1314,10 @@
<span class="sd"> :type ignore_if_missing: boolean</span>
<span class="sd"> :return:</span>
<span class="sd"> """</span>
-
- <span class="k">assert</span> <span class="s1">'.'</span> <span class="ow">in</span> <span class="n">deletion_dataset_table</span><span class="p">,</span> <span class="p">(</span>
- <span class="s1">'Expected deletion_dataset_table in the format of '</span>
- <span class="s1">'<dataset>.<table>. Got: </span><span class="si">{}</span><span class="s1">'</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">deletion_dataset_table</span><span class="p">)</span>
+ <span class="k">if</span> <span class="s1">'.'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">deletion_dataset_table</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+ <span class="s1">'Expected deletion_dataset_table name in the format of '</span>
+ <span class="s1">'<dataset>.<table>. Got: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">deletion_dataset_table</span><span class="p">))</span>
<span class="n">deletion_project</span><span class="p">,</span> <span class="n">deletion_dataset</span><span class="p">,</span> <span class="n">deletion_table</span> <span class="o">=</span> \
<span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">deletion_dataset_table</span><span class="p">,</span>
<span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">)</span>
@@ -1412,10 +1445,34 @@
<span class="c1"># if view is already in access, do nothing.</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
<span class="s1">'Table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> already has authorized view access to </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1"> dataset.'</span><span class="p">,</span>
- <span class="n">view_project</span><span class="p">,</span> <span class="n">view_dataset</span><span class="p">,</span> <span class="n">view_table</span><span class="p">,</span> <span class="n">source_project</span><span class="p">,</span>
- <span class="n">source_dataset</span><span class="p">)</span>
+ <span class="n">view_project</span><span class="p">,</span> <span class="n">view_dataset</span><span class="p">,</span> <span class="n">view_table</span><span class="p">,</span> <span class="n">source_project</span><span class="p">,</span> <span class="n">source_dataset</span><span class="p">)</span>
<span class="k">return</span> <span class="n">source_dataset_resource</span>
+ <span class="k">def</span> <span class="nf">delete_dataset</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Delete a dataset of Big query in your project.</span>
+<span class="sd"> :param project_id: The name of the project where we have the dataset .</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :param dataset_id: The dataset to be delete.</span>
+<span class="sd"> :type dataset_id: str</span>
+<span class="sd"> :return:</span>
+<span class="sd"> """</span>
+ <span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span> <span class="k">if</span> <span class="n">project_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Deleting from project: </span><span class="si">%s</span><span class="s1"> Dataset:</span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">)</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">datasets</span><span class="p">()</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Dataset deleted successfully: In project </span><span class="si">%s</span><span class="s1"> '</span>
+ <span class="s1">'Dataset </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">)</span>
+
+ <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'BigQuery job failed. Error was: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">err</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
+ <span class="p">)</span>
+
<span class="k">class</span> <span class="nc">BigQueryCursor</span><span class="p">(</span><span class="n">BigQueryBaseCursor</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -1617,14 +1674,17 @@
<span class="k">elif</span> <span class="n">bq_type</span> <span class="o">==</span> <span class="s1">'FLOAT'</span> <span class="ow">or</span> <span class="n">bq_type</span> <span class="o">==</span> <span class="s1">'TIMESTAMP'</span><span class="p">:</span>
<span class="k">return</span> <span class="nb">float</span><span class="p">(</span><span class="n">string_field</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">bq_type</span> <span class="o">==</span> <span class="s1">'BOOLEAN'</span><span class="p">:</span>
- <span class="k">assert</span> <span class="n">string_field</span> <span class="ow">in</span> <span class="nb">set</span><span class="p">([</span><span class="s1">'true'</span><span class="p">,</span> <span class="s1">'false'</span><span class="p">])</span>
+ <span class="k">if</span> <span class="n">string_field</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'true'</span><span class="p">,</span> <span class="s1">'false'</span><span class="p">]:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"</span><span class="si">{}</span><span class="s2"> must have value 'true' or 'false'"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">string_field</span><span class="p">))</span>
<span class="k">return</span> <span class="n">string_field</span> <span class="o">==</span> <span class="s1">'true'</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="n">string_field</span>
<span class="k">def</span> <span class="nf">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="p">,</span> <span class="n">default_project_id</span><span class="p">,</span> <span class="n">var_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
- <span class="k">assert</span> <span class="n">default_project_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">,</span> <span class="s2">"INTERNAL: No default project is specified"</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">default_project_id</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"INTERNAL: No default project is specified"</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">var_print</span><span class="p">(</span><span class="n">var_name</span><span class="p">):</span>
<span class="k">if</span> <span class="n">var_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
@@ -1636,7 +1696,6 @@
<span class="k">raise</span> <span class="ne">Exception</span><span class="p">((</span><span class="s1">'</span><span class="si">{var}</span><span class="s1">Use either : or . to specify project '</span>
<span class="s1">'got </span><span class="si">{input}</span><span class="s1">'</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="n">var</span><span class="o">=</span><span class="n">var_print</span><span class="p">(</span><span class="n">var_name</span><span class="p">),</span> <span class="nb">input</span><span class="o">=</span><span class="n">table_input</span><span class="p">))</span>
-
<span class="n">cmpt</span> <span class="o">=</span> <span class="n">table_input</span><span class="o">.</span><span class="n">rsplit</span><span class="p">(</span><span class="s1">':'</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
<span class="n">project_id</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">rest</span> <span class="o">=</span> <span class="n">table_input</span>
@@ -1654,8 +1713,10 @@
<span class="n">cmpt</span> <span class="o">=</span> <span class="n">rest</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">cmpt</span><span class="p">)</span> <span class="o">==</span> <span class="mi">3</span><span class="p">:</span>
- <span class="k">assert</span> <span class="n">project_id</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">,</span> <span class="p">(</span><span class="s2">"</span><span class="si">{var}</span><span class="s2">Use either : or . to specify project"</span>
- <span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">var</span><span class="o">=</span><span class="n">var_print</span><span class="p">(</span><span class="n">var_name</span><span class="p">))</span>
+ <span class="k">if</span> <span class="n">project_id</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+ <span class="s2">"</span><span class="si">{var}</span><span class="s2">Use either : or . to specify project"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">var</span><span class="o">=</span><span class="n">var_print</span><span class="p">(</span><span class="n">var_name</span><span class="p">)))</span>
<span class="n">project_id</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">dataset_id</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
<span class="n">table_id</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span>
@@ -1685,12 +1746,7 @@
<span class="c1"># if it is a partitioned table ($ is in the table name) add partition load option</span>
<span class="n">time_partitioning_out</span> <span class="o">=</span> <span class="p">{}</span>
<span class="k">if</span> <span class="n">destination_dataset_table</span> <span class="ow">and</span> <span class="s1">'$'</span> <span class="ow">in</span> <span class="n">destination_dataset_table</span><span class="p">:</span>
- <span class="k">assert</span> <span class="ow">not</span> <span class="n">time_partitioning_in</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'field'</span><span class="p">),</span> <span class="p">(</span>
- <span class="s2">"Cannot specify field partition and partition name "</span>
- <span class="s2">"(dataset.table$partition) at the same time"</span>
- <span class="p">)</span>
<span class="n">time_partitioning_out</span><span class="p">[</span><span class="s1">'type'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'DAY'</span>
-
<span class="n">time_partitioning_out</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">time_partitioning_in</span><span class="p">)</span>
<span class="k">return</span> <span class="n">time_partitioning_out</span>
</pre></div>
@@ -1723,20 +1779,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/cassandra_hook.html b/_modules/airflow/contrib/hooks/cassandra_hook.html
index 01142f1..46f8c38 100644
--- a/_modules/airflow/contrib/hooks/cassandra_hook.html
+++ b/_modules/airflow/contrib/hooks/cassandra_hook.html
@@ -371,20 +371,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/cloudant_hook.html b/_modules/airflow/contrib/hooks/cloudant_hook.html
index 5e762ea..42cf9e5 100644
--- a/_modules/airflow/contrib/hooks/cloudant_hook.html
+++ b/_modules/airflow/contrib/hooks/cloudant_hook.html
@@ -272,20 +272,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/databricks_hook.html b/_modules/airflow/contrib/hooks/databricks_hook.html
index 0d21a4a..0493901 100644
--- a/_modules/airflow/contrib/hooks/databricks_hook.html
+++ b/_modules/airflow/contrib/hooks/databricks_hook.html
@@ -185,6 +185,7 @@
<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
<span class="kn">from</span> <span class="nn">requests</span> <span class="k">import</span> <span class="n">exceptions</span> <span class="k">as</span> <span class="n">requests_exceptions</span>
<span class="kn">from</span> <span class="nn">requests.auth</span> <span class="k">import</span> <span class="n">AuthBase</span>
+<span class="kn">from</span> <span class="nn">time</span> <span class="k">import</span> <span class="n">sleep</span>
<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
@@ -193,6 +194,9 @@
<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
<span class="kn">import</span> <span class="nn">urlparse</span>
+<span class="n">RESTART_CLUSTER_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s2">"POST"</span><span class="p">,</span> <span class="s2">"api/2.0/clusters/restart"</span><span class="p">)</span>
+<span class="n">START_CLUSTER_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s2">"POST"</span><span class="p">,</span> <span class="s2">"api/2.0/clusters/start"</span><span class="p">)</span>
+<span class="n">TERMINATE_CLUSTER_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s2">"POST"</span><span class="p">,</span> <span class="s2">"api/2.0/clusters/delete"</span><span class="p">)</span>
<span class="n">SUBMIT_RUN_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'POST'</span><span class="p">,</span> <span class="s1">'api/2.0/jobs/runs/submit'</span><span class="p">)</span>
<span class="n">GET_RUN_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'GET'</span><span class="p">,</span> <span class="s1">'api/2.0/jobs/runs/get'</span><span class="p">)</span>
@@ -208,7 +212,8 @@
<span class="bp">self</span><span class="p">,</span>
<span class="n">databricks_conn_id</span><span class="o">=</span><span class="s1">'databricks_default'</span><span class="p">,</span>
<span class="n">timeout_seconds</span><span class="o">=</span><span class="mi">180</span><span class="p">,</span>
- <span class="n">retry_limit</span><span class="o">=</span><span class="mi">3</span><span class="p">):</span>
+ <span class="n">retry_limit</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span>
+ <span class="n">retry_delay</span><span class="o">=</span><span class="mf">1.0</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> :param databricks_conn_id: The name of the databricks connection to use.</span>
<span class="sd"> :type databricks_conn_id: string</span>
@@ -218,12 +223,17 @@
<span class="sd"> :param retry_limit: The number of times to retry the connection in case of</span>
<span class="sd"> service outages.</span>
<span class="sd"> :type retry_limit: int</span>
+<span class="sd"> :param retry_delay: The number of seconds to wait between retries (it</span>
+<span class="sd"> might be a floating point number).</span>
+<span class="sd"> :type retry_delay: float</span>
<span class="sd"> """</span>
<span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn_id</span> <span class="o">=</span> <span class="n">databricks_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">databricks_conn_id</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">timeout_seconds</span> <span class="o">=</span> <span class="n">timeout_seconds</span>
- <span class="k">assert</span> <span class="n">retry_limit</span> <span class="o">>=</span> <span class="mi">1</span><span class="p">,</span> <span class="s1">'Retry limit must be greater than equal to 1'</span>
+ <span class="k">if</span> <span class="n">retry_limit</span> <span class="o"><</span> <span class="mi">1</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'Retry limit must be greater than equal to 1'</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">retry_limit</span> <span class="o">=</span> <span class="n">retry_limit</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">retry_delay</span> <span class="o">=</span> <span class="n">retry_delay</span>
<span class="k">def</span> <span class="nf">_parse_host</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">host</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -278,7 +288,8 @@
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Unexpected HTTP Method: '</span> <span class="o">+</span> <span class="n">method</span><span class="p">)</span>
- <span class="k">for</span> <span class="n">attempt_num</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">retry_limit</span> <span class="o">+</span> <span class="mi">1</span><span class="p">):</span>
+ <span class="n">attempt_num</span> <span class="o">=</span> <span class="mi">1</span>
+ <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">response</span> <span class="o">=</span> <span class="n">request_func</span><span class="p">(</span>
<span class="n">url</span><span class="p">,</span>
@@ -286,21 +297,29 @@
<span class="n">auth</span><span class="o">=</span><span class="n">auth</span><span class="p">,</span>
<span class="n">headers</span><span class="o">=</span><span class="n">USER_AGENT_HEADER</span><span class="p">,</span>
<span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout_seconds</span><span class="p">)</span>
- <span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="n">requests</span><span class="o">.</span><span class="n">codes</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
- <span class="k">else</span><span class="p">:</span>
+ <span class="n">response</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
+ <span class="k">except</span> <span class="n">requests_exceptions</span><span class="o">.</span><span class="n">RequestException</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">_retryable_error</span><span class="p">(</span><span class="n">e</span><span class="p">):</span>
<span class="c1"># In this case, the user probably made a mistake.</span>
<span class="c1"># Don't retry.</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Response: </span><span class="si">{0}</span><span class="s1">, Status Code: </span><span class="si">{1}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
- <span class="n">response</span><span class="o">.</span><span class="n">content</span><span class="p">,</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span><span class="p">))</span>
- <span class="k">except</span> <span class="p">(</span><span class="n">requests_exceptions</span><span class="o">.</span><span class="n">ConnectionError</span><span class="p">,</span>
- <span class="n">requests_exceptions</span><span class="o">.</span><span class="n">Timeout</span><span class="p">)</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
- <span class="s1">'Attempt </span><span class="si">%s</span><span class="s1"> API Request to Databricks failed with reason: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
- <span class="n">attempt_num</span><span class="p">,</span> <span class="n">e</span>
- <span class="p">)</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">((</span><span class="s1">'API requests to Databricks failed </span><span class="si">{}</span><span class="s1"> times. '</span> <span class="o">+</span>
- <span class="s1">'Giving up.'</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">retry_limit</span><span class="p">))</span>
+ <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="o">.</span><span class="n">content</span><span class="p">,</span> <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="o">.</span><span class="n">status_code</span><span class="p">))</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">_log_request_error</span><span class="p">(</span><span class="n">attempt_num</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">attempt_num</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">retry_limit</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">((</span><span class="s1">'API requests to Databricks failed </span><span class="si">{}</span><span class="s1"> times. '</span> <span class="o">+</span>
+ <span class="s1">'Giving up.'</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">retry_limit</span><span class="p">))</span>
+
+ <span class="n">attempt_num</span> <span class="o">+=</span> <span class="mi">1</span>
+ <span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">retry_delay</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_log_request_error</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attempt_num</span><span class="p">,</span> <span class="n">error</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
+ <span class="s1">'Attempt </span><span class="si">%s</span><span class="s1"> API Request to Databricks failed with reason: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">attempt_num</span><span class="p">,</span> <span class="n">error</span>
+ <span class="p">)</span>
<div class="viewcode-block" id="DatabricksHook.submit_run"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.databricks_hook.DatabricksHook.submit_run">[docs]</a> <span class="k">def</span> <span class="nf">submit_run</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -331,7 +350,22 @@
<span class="k">def</span> <span class="nf">cancel_run</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">run_id</span><span class="p">):</span>
<span class="n">json</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'run_id'</span><span class="p">:</span> <span class="n">run_id</span><span class="p">}</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">CANCEL_RUN_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">CANCEL_RUN_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">restart_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">RESTART_CLUSTER_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">start_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">START_CLUSTER_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">terminate_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">TERMINATE_CLUSTER_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span></div>
+
+
+<span class="k">def</span> <span class="nf">_retryable_error</span><span class="p">(</span><span class="n">exception</span><span class="p">):</span>
+ <span class="k">return</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">exception</span><span class="p">,</span> <span class="n">requests_exceptions</span><span class="o">.</span><span class="n">ConnectionError</span><span class="p">)</span> \
+ <span class="ow">or</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">exception</span><span class="p">,</span> <span class="n">requests_exceptions</span><span class="o">.</span><span class="n">Timeout</span><span class="p">)</span> \
+ <span class="ow">or</span> <span class="n">exception</span><span class="o">.</span><span class="n">response</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">exception</span><span class="o">.</span><span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">>=</span> <span class="mi">500</span>
<span class="n">RUN_LIFE_CYCLE_STATES</span> <span class="o">=</span> <span class="p">[</span>
@@ -417,20 +451,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/datadog_hook.html b/_modules/airflow/contrib/hooks/datadog_hook.html
index bb0a05f..cfa1ad2 100644
--- a/_modules/airflow/contrib/hooks/datadog_hook.html
+++ b/_modules/airflow/contrib/hooks/datadog_hook.html
@@ -331,20 +331,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/datastore_hook.html b/_modules/airflow/contrib/hooks/datastore_hook.html
index 29aa16f..5868383 100644
--- a/_modules/airflow/contrib/hooks/datastore_hook.html
+++ b/_modules/airflow/contrib/hooks/datastore_hook.html
@@ -395,20 +395,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/discord_webhook_hook.html b/_modules/airflow/contrib/hooks/discord_webhook_hook.html
index b82d631..6622022 100644
--- a/_modules/airflow/contrib/hooks/discord_webhook_hook.html
+++ b/_modules/airflow/contrib/hooks/discord_webhook_hook.html
@@ -329,20 +329,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/emr_hook.html b/_modules/airflow/contrib/hooks/emr_hook.html
index 7aa2a43..e2e6873 100644
--- a/_modules/airflow/contrib/hooks/emr_hook.html
+++ b/_modules/airflow/contrib/hooks/emr_hook.html
@@ -212,20 +212,7 @@
<span class="n">config</span> <span class="o">=</span> <span class="n">emr_conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">config</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">job_flow_overrides</span><span class="p">)</span>
- <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">run_job_flow</span><span class="p">(</span>
- <span class="n">Name</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Name'</span><span class="p">),</span>
- <span class="n">LogUri</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'LogUri'</span><span class="p">),</span>
- <span class="n">ReleaseLabel</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'ReleaseLabel'</span><span class="p">),</span>
- <span class="n">Instances</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Instances'</span><span class="p">),</span>
- <span class="n">Steps</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Steps'</span><span class="p">,</span> <span class="p">[]),</span>
- <span class="n">BootstrapActions</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'BootstrapActions'</span><span class="p">,</span> <span class="p">[]),</span>
- <span class="n">Applications</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Applications'</span><span class="p">),</span>
- <span class="n">Configurations</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Configurations'</span><span class="p">,</span> <span class="p">[]),</span>
- <span class="n">VisibleToAllUsers</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'VisibleToAllUsers'</span><span class="p">),</span>
- <span class="n">JobFlowRole</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'JobFlowRole'</span><span class="p">),</span>
- <span class="n">ServiceRole</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'ServiceRole'</span><span class="p">),</span>
- <span class="n">Tags</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Tags'</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">run_job_flow</span><span class="p">(</span><span class="o">**</span><span class="n">config</span><span class="p">)</span>
<span class="k">return</span> <span class="n">response</span></div></div>
</pre></div>
@@ -258,20 +245,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/fs_hook.html b/_modules/airflow/contrib/hooks/fs_hook.html
index 7b01482..ef59c77 100644
--- a/_modules/airflow/contrib/hooks/fs_hook.html
+++ b/_modules/airflow/contrib/hooks/fs_hook.html
@@ -235,20 +235,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/ftp_hook.html b/_modules/airflow/contrib/hooks/ftp_hook.html
index 88879d9..3b25ec7 100644
--- a/_modules/airflow/contrib/hooks/ftp_hook.html
+++ b/_modules/airflow/contrib/hooks/ftp_hook.html
@@ -309,7 +309,11 @@
<span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
<span class="n">conn</span><span class="o">.</span><span class="n">rmd</span><span class="p">(</span><span class="n">path</span><span class="p">)</span></div>
-<div class="viewcode-block" id="FTPHook.retrieve_file"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPHook.retrieve_file">[docs]</a> <span class="k">def</span> <span class="nf">retrieve_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">remote_full_path</span><span class="p">,</span> <span class="n">local_full_path_or_buffer</span><span class="p">):</span>
+<div class="viewcode-block" id="FTPHook.retrieve_file"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPHook.retrieve_file">[docs]</a> <span class="k">def</span> <span class="nf">retrieve_file</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">remote_full_path</span><span class="p">,</span>
+ <span class="n">local_full_path_or_buffer</span><span class="p">,</span>
+ <span class="n">callback</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Transfers the remote file to a local location.</span>
@@ -322,23 +326,59 @@
<span class="sd"> :param local_full_path_or_buffer: full path to the local file or a</span>
<span class="sd"> file-like buffer</span>
<span class="sd"> :type local_full_path_or_buffer: str or file-like buffer</span>
+<span class="sd"> :param callback: callback which is called each time a block of data</span>
+<span class="sd"> is read. if you do not use a callback, these blocks will be written</span>
+<span class="sd"> to the file or buffer passed in. if you do pass in a callback, note</span>
+<span class="sd"> that writing to a file or buffer will need to be handled inside the</span>
+<span class="sd"> callback.</span>
+<span class="sd"> [default: output_handle.write()]</span>
+<span class="sd"> :type callback: callable</span>
+
+<span class="sd"> Example::</span>
+<span class="sd"> hook = FTPHook(ftp_conn_id='my_conn')</span>
+
+<span class="sd"> remote_path = '/path/to/remote/file'</span>
+<span class="sd"> local_path = '/path/to/local/file'</span>
+
+<span class="sd"> # with a custom callback (in this case displaying progress on each read)</span>
+<span class="sd"> def print_progress(percent_progress):</span>
+<span class="sd"> self.log.info('Percent Downloaded: %s%%' % percent_progress)</span>
+
+<span class="sd"> total_downloaded = 0</span>
+<span class="sd"> total_file_size = hook.get_size(remote_path)</span>
+<span class="sd"> output_handle = open(local_path, 'wb')</span>
+<span class="sd"> def write_to_file_with_progress(data):</span>
+<span class="sd"> total_downloaded += len(data)</span>
+<span class="sd"> output_handle.write(data)</span>
+<span class="sd"> percent_progress = (total_downloaded / total_file_size) * 100</span>
+<span class="sd"> print_progress(percent_progress)</span>
+<span class="sd"> hook.retrieve_file(remote_path, None, callback=write_to_file_with_progress)</span>
+
+<span class="sd"> # without a custom callback data is written to the local_path</span>
+<span class="sd"> hook.retrieve_file(remote_path, local_path)</span>
<span class="sd"> """</span>
<span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
<span class="n">is_path</span> <span class="o">=</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">local_full_path_or_buffer</span><span class="p">,</span> <span class="n">basestring</span><span class="p">)</span>
- <span class="k">if</span> <span class="n">is_path</span><span class="p">:</span>
- <span class="n">output_handle</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">local_full_path_or_buffer</span><span class="p">,</span> <span class="s1">'wb'</span><span class="p">)</span>
+ <span class="c1"># without a callback, default to writing to a user-provided file or</span>
+ <span class="c1"># file-like buffer</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">callback</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">is_path</span><span class="p">:</span>
+ <span class="n">output_handle</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">local_full_path_or_buffer</span><span class="p">,</span> <span class="s1">'wb'</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">output_handle</span> <span class="o">=</span> <span class="n">local_full_path_or_buffer</span>
+ <span class="n">callback</span> <span class="o">=</span> <span class="n">output_handle</span><span class="o">.</span><span class="n">write</span>
<span class="k">else</span><span class="p">:</span>
- <span class="n">output_handle</span> <span class="o">=</span> <span class="n">local_full_path_or_buffer</span>
+ <span class="n">output_handle</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">remote_path</span><span class="p">,</span> <span class="n">remote_file_name</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">remote_full_path</span><span class="p">)</span>
<span class="n">conn</span><span class="o">.</span><span class="n">cwd</span><span class="p">(</span><span class="n">remote_path</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Retrieving file from FTP: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">remote_full_path</span><span class="p">)</span>
- <span class="n">conn</span><span class="o">.</span><span class="n">retrbinary</span><span class="p">(</span><span class="s1">'RETR </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">remote_file_name</span><span class="p">,</span> <span class="n">output_handle</span><span class="o">.</span><span class="n">write</span><span class="p">)</span>
+ <span class="n">conn</span><span class="o">.</span><span class="n">retrbinary</span><span class="p">(</span><span class="s1">'RETR </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">remote_file_name</span><span class="p">,</span> <span class="n">callback</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Finished retrieving file from FTP: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">remote_full_path</span><span class="p">)</span>
- <span class="k">if</span> <span class="n">is_path</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">is_path</span> <span class="ow">and</span> <span class="n">output_handle</span><span class="p">:</span>
<span class="n">output_handle</span><span class="o">.</span><span class="n">close</span><span class="p">()</span></div>
<div class="viewcode-block" id="FTPHook.store_file"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPHook.store_file">[docs]</a> <span class="k">def</span> <span class="nf">store_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">remote_full_path</span><span class="p">,</span> <span class="n">local_full_path_or_buffer</span><span class="p">):</span>
@@ -390,7 +430,13 @@
<span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
<span class="k">return</span> <span class="n">conn</span><span class="o">.</span><span class="n">rename</span><span class="p">(</span><span class="n">from_name</span><span class="p">,</span> <span class="n">to_name</span><span class="p">)</span></div>
- <span class="k">def</span> <span class="nf">get_mod_time</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+<div class="viewcode-block" id="FTPHook.get_mod_time"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPHook.get_mod_time">[docs]</a> <span class="k">def</span> <span class="nf">get_mod_time</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns a datetime object representing the last time the file was modified</span>
+
+<span class="sd"> :param path: remote file path</span>
+<span class="sd"> :type path: string</span>
+<span class="sd"> """</span>
<span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
<span class="n">ftp_mdtm</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">sendcmd</span><span class="p">(</span><span class="s1">'MDTM '</span> <span class="o">+</span> <span class="n">path</span><span class="p">)</span>
<span class="n">time_val</span> <span class="o">=</span> <span class="n">ftp_mdtm</span><span class="p">[</span><span class="mi">4</span><span class="p">:]</span>
@@ -400,6 +446,16 @@
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
<span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">time_val</span><span class="p">,</span> <span class="s1">'%Y%m</span><span class="si">%d</span><span class="s1">%H%M%S'</span><span class="p">)</span></div>
+<div class="viewcode-block" id="FTPHook.get_size"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPHook.get_size">[docs]</a> <span class="k">def</span> <span class="nf">get_size</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns the size of a file (in bytes)</span>
+
+<span class="sd"> :param path: remote file path</span>
+<span class="sd"> :type path: string</span>
+<span class="sd"> """</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">conn</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="n">path</span><span class="p">)</span></div></div>
+
<div class="viewcode-block" id="FTPSHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPSHook">[docs]</a><span class="k">class</span> <span class="nc">FTPSHook</span><span class="p">(</span><span class="n">FTPHook</span><span class="p">):</span>
@@ -448,20 +504,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/gcp_api_base_hook.html b/_modules/airflow/contrib/hooks/gcp_api_base_hook.html
index 78b729d..22ddd66 100644
--- a/_modules/airflow/contrib/hooks/gcp_api_base_hook.html
+++ b/_modules/airflow/contrib/hooks/gcp_api_base_hook.html
@@ -248,7 +248,7 @@
<span class="k">elif</span> <span class="n">key_path</span><span class="p">:</span>
<span class="c1"># Get credentials from a JSON file.</span>
<span class="k">if</span> <span class="n">key_path</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s1">'.json'</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Getting connection using a JSON key file.'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'Getting connection using JSON key file </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">key_path</span><span class="p">)</span>
<span class="n">credentials</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">google</span><span class="o">.</span><span class="n">oauth2</span><span class="o">.</span><span class="n">service_account</span><span class="o">.</span><span class="n">Credentials</span><span class="o">.</span><span class="n">from_service_account_file</span><span class="p">(</span>
<span class="n">key_path</span><span class="p">,</span> <span class="n">scopes</span><span class="o">=</span><span class="n">scopes</span><span class="p">)</span>
@@ -341,20 +341,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/gcp_container_hook.html b/_modules/airflow/contrib/hooks/gcp_container_hook.html
index 4cb7114..983c362 100644
--- a/_modules/airflow/contrib/hooks/gcp_container_hook.html
+++ b/_modules/airflow/contrib/hooks/gcp_container_hook.html
@@ -208,6 +208,7 @@
<span class="k">def</span> <span class="nf">_dict_to_proto</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">py_dict</span><span class="p">,</span> <span class="n">proto</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Converts a python dictionary to the proto supplied</span>
+
<span class="sd"> :param py_dict: The dictionary to convert</span>
<span class="sd"> :type py_dict: dict</span>
<span class="sd"> :param proto: The proto object to merge with dictionary</span>
@@ -223,6 +224,7 @@
<span class="sd">"""</span>
<span class="sd"> Given an operation, continuously fetches the status from Google Cloud until either</span>
<span class="sd"> completion or an error occurring</span>
+
<span class="sd"> :param operation: The Operation to wait for</span>
<span class="sd"> :type operation: A google.cloud.container_V1.gapic.enums.Operator</span>
<span class="sd"> :return: A new, updated operation fetched from Google Cloud</span>
@@ -243,6 +245,7 @@
<div class="viewcode-block" id="GKEClusterHook.get_operation"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_container_hook.GKEClusterHook.get_operation">[docs]</a> <span class="k">def</span> <span class="nf">get_operation</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Fetches the operation from Google Cloud</span>
+
<span class="sd"> :param operation_name: Name of operation to fetch</span>
<span class="sd"> :type operation_name: str</span>
<span class="sd"> :return: The new, updated operation from Google Cloud</span>
@@ -355,6 +358,7 @@
<div class="viewcode-block" id="GKEClusterHook.get_cluster"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_container_hook.GKEClusterHook.get_cluster">[docs]</a> <span class="k">def</span> <span class="nf">get_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">retry</span><span class="o">=</span><span class="n">DEFAULT</span><span class="p"> [...]
<span class="sd">"""</span>
<span class="sd"> Gets details of specified cluster</span>
+
<span class="sd"> :param name: The name of the cluster to retrieve</span>
<span class="sd"> :type name: str</span>
<span class="sd"> :param retry: A retry object used to retry requests. If None is specified,</span>
@@ -406,20 +410,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/gcp_dataflow_hook.html b/_modules/airflow/contrib/hooks/gcp_dataflow_hook.html
index 808d22f..cf99744 100644
--- a/_modules/airflow/contrib/hooks/gcp_dataflow_hook.html
+++ b/_modules/airflow/contrib/hooks/gcp_dataflow_hook.html
@@ -178,6 +178,7 @@
<span class="c1"># specific language governing permissions and limitations</span>
<span class="c1"># under the License.</span>
<span class="kn">import</span> <span class="nn">json</span>
+<span class="kn">import</span> <span class="nn">re</span>
<span class="kn">import</span> <span class="nn">select</span>
<span class="kn">import</span> <span class="nn">subprocess</span>
<span class="kn">import</span> <span class="nn">time</span>
@@ -194,12 +195,13 @@
<span class="k">class</span> <span class="nc">_DataflowJob</span><span class="p">(</span><span class="n">LoggingMixin</span><span class="p">):</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">,</span> <span class="n">project_number</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">location</span><span class="p">,</span> <span class="n">poll_sleep</span><span class="o">=</span><span class="mi">10</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">,</span> <span class="n">project_number</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">location</span><span class="p">,</span> <span class="n">poll_sleep</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span>
+ <span class="n">job_id</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_dataflow</span> <span class="o">=</span> <span class="n">dataflow</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_project_number</span> <span class="o">=</span> <span class="n">project_number</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_job_name</span> <span class="o">=</span> <span class="n">name</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_job_location</span> <span class="o">=</span> <span class="n">location</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">_job_id</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_job_id</span> <span class="o">=</span> <span class="n">job_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_job</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_job</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_poll_sleep</span> <span class="o">=</span> <span class="n">poll_sleep</span>
@@ -207,7 +209,7 @@
<span class="n">jobs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_dataflow</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">locations</span><span class="p">()</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span><span class="o">.</span><span class="n">list</span><span class="p">(</span>
<span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_project_number</span><span class="p">,</span>
<span class="n">location</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_job_location</span>
- <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">num_retries</span><span class="o">=</span><span class="mi">5</span><span class="p">)</span>
<span class="k">for</span> <span class="n">job</span> <span class="ow">in</span> <span class="n">jobs</span><span class="p">[</span><span class="s1">'jobs'</span><span class="p">]:</span>
<span class="k">if</span> <span class="n">job</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job_name</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_job_id</span> <span class="o">=</span> <span class="n">job</span><span class="p">[</span><span class="s1">'id'</span><span class="p">]</span>
@@ -215,13 +217,15 @@
<span class="k">return</span> <span class="kc">None</span>
<span class="k">def</span> <span class="nf">_get_job</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job_name</span><span class="p">:</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job_id</span><span class="p">:</span>
+ <span class="n">job</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_dataflow</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">locations</span><span class="p">()</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_project_number</span><span class="p">,</span>
+ <span class="n">location</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_job_location</span><span class="p">,</span>
+ <span class="n">jobId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_job_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">num_retries</span><span class="o">=</span><span class="mi">5</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_job_name</span><span class="p">:</span>
<span class="n">job</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_job_id_from_name</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
- <span class="n">job</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_dataflow</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
- <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_project_number</span><span class="p">,</span>
- <span class="n">jobId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_job_id</span>
- <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s1">'Missing both dataflow job ID and name.'</span><span class="p">)</span>
<span class="k">if</span> <span class="n">job</span> <span class="ow">and</span> <span class="s1">'currentState'</span> <span class="ow">in</span> <span class="n">job</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
@@ -284,36 +288,50 @@
<span class="k">def</span> <span class="nf">_line</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">fd</span><span class="p">):</span>
<span class="k">if</span> <span class="n">fd</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">stderr</span><span class="o">.</span><span class="n">fileno</span><span class="p">():</span>
- <span class="n">lines</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">stderr</span><span class="o">.</span><span class="n">readlines</span><span class="p">()</span>
- <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">lines</span><span class="p">:</span>
+ <span class="n">line</span> <span class="o">=</span> <span class="sa">b</span><span class="s1">''</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">stderr</span><span class="o">.</span><span class="n">readlines</span><span class="p">())</span>
+ <span class="k">if</span> <span class="n">line</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">line</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
- <span class="k">if</span> <span class="n">lines</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">lines</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+ <span class="k">return</span> <span class="n">line</span>
<span class="k">if</span> <span class="n">fd</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">fileno</span><span class="p">():</span>
- <span class="n">line</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">readline</span><span class="p">()</span>
+ <span class="n">line</span> <span class="o">=</span> <span class="sa">b</span><span class="s1">''</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">readlines</span><span class="p">())</span>
+ <span class="k">if</span> <span class="n">line</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">line</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
<span class="k">return</span> <span class="n">line</span>
<span class="nd">@staticmethod</span>
<span class="k">def</span> <span class="nf">_extract_job</span><span class="p">(</span><span class="n">line</span><span class="p">):</span>
- <span class="k">if</span> <span class="n">line</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
- <span class="k">if</span> <span class="n">line</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"Submitted job: "</span><span class="p">):</span>
- <span class="k">return</span> <span class="n">line</span><span class="p">[</span><span class="mi">15</span><span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+ <span class="c1"># Job id info: https://goo.gl/SE29y9.</span>
+ <span class="n">job_id_pattern</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span>
+ <span class="sa">b</span><span class="s1">'.*console.cloud.google.com/dataflow.*/jobs/([a-z|0-9|A-Z|\-|\_]+).*'</span><span class="p">)</span>
+ <span class="n">matched_job</span> <span class="o">=</span> <span class="n">job_id_pattern</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">line</span> <span class="ow">or</span> <span class="s1">''</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">matched_job</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">matched_job</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">wait_for_done</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">reads</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">stderr</span><span class="o">.</span><span class="n">fileno</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">fileno</span><span c [...]
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Start waiting for DataFlow process to complete."</span><span class="p">)</span>
- <span class="k">while</span> <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">poll</span><span class="p">()</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">job_id</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="c1"># Make sure logs are processed regardless whether the subprocess is</span>
+ <span class="c1"># terminated.</span>
+ <span class="n">process_ends</span> <span class="o">=</span> <span class="kc">False</span>
+ <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
<span class="n">ret</span> <span class="o">=</span> <span class="n">select</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">reads</span><span class="p">,</span> <span class="p">[],</span> <span class="p">[],</span> <span class="mi">5</span><span class="p">)</span>
<span class="k">if</span> <span class="n">ret</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">for</span> <span class="n">fd</span> <span class="ow">in</span> <span class="n">ret</span><span class="p">[</span><span class="mi">0</span><span class="p">]:</span>
<span class="n">line</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_line</span><span class="p">(</span><span class="n">fd</span><span class="p">)</span>
<span class="k">if</span> <span class="n">line</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="n">line</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
+ <span class="n">job_id</span> <span class="o">=</span> <span class="n">job_id</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extract_job</span><span class="p">(</span><span class="n">line</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Waiting for DataFlow process to complete."</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">process_ends</span><span class="p">:</span>
+ <span class="k">break</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">poll</span><span class="p">()</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="c1"># Mark process completion but allows its outputs to be consumed.</span>
+ <span class="n">process_ends</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">returncode</span> <span class="ow">is</span> <span class="ow">not</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s2">"DataFlow failed with return code </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_proc</span><span class="o">.</span><span class="n">returncode</span><span class="p">))</span>
+ <span class="k">return</span> <span class="n">job_id</span>
<div class="viewcode-block" id="DataFlowHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_dataflow_hook.DataFlowHook">[docs]</a><span class="k">class</span> <span class="nc">DataFlowHook</span><span class="p">(</span><span class="n">GoogleCloudBaseHook</span><span class="p">):</span>
@@ -327,7 +345,7 @@
<div class="viewcode-block" id="DataFlowHook.get_conn"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_dataflow_hook.DataFlowHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
-<span class="sd"> Returns a Google Cloud Storage service object.</span>
+<span class="sd"> Returns a Google Cloud Dataflow service object.</span>
<span class="sd"> """</span>
<span class="n">http_authorized</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
<span class="k">return</span> <span class="n">build</span><span class="p">(</span>
@@ -338,9 +356,10 @@
<span class="n">variables</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_set_variables</span><span class="p">(</span><span class="n">variables</span><span class="p">)</span>
<span class="n">cmd</span> <span class="o">=</span> <span class="n">command_prefix</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_cmd</span><span class="p">(</span><span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span>
<span class="n">label_formatter</span><span class="p">)</span>
- <span class="n">_Dataflow</span><span class="p">(</span><span class="n">cmd</span><span class="p">)</span><span class="o">.</span><span class="n">wait_for_done</span><span class="p">()</span>
+ <span class="n">job_id</span> <span class="o">=</span> <span class="n">_Dataflow</span><span class="p">(</span><span class="n">cmd</span><span class="p">)</span><span class="o">.</span><span class="n">wait_for_done</span><span class="p">()</span>
<span class="n">_DataflowJob</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">(),</span> <span class="n">variables</span><span class="p">[</span><span class="s1">'project'</span><span class="p">],</span> <span class="n">name</span><span class="p">,</span>
- <span class="n">variables</span><span class="p">[</span><span class="s1">'region'</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">poll_sleep</span><span class="p">)</span><span class="o">.</span><span class="n">wait_for_done</span><span class="p">()</span>
+ <span class="n">variables</span><span class="p">[</span><span class="s1">'region'</span><span class="p">],</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">poll_sleep</span><span class="p">,</span> <span class="n">job_id</span><span class="p">)</span><span class="o">.</span><span class="n">wait_for_done</span><span class="p">()</span>
<span class="nd">@staticmethod</span>
<span class="k">def</span> <span class="nf">_set_variables</span><span class="p">(</span><span class="n">variables</span><span class="p">):</span>
@@ -352,10 +371,7 @@
<span class="k">def</span> <span class="nf">start_java_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">,</span> <span class="n">job_class</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">append_job_name</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
- <span class="k">if</span> <span class="n">append_job_name</span><span class="p">:</span>
- <span class="n">name</span> <span class="o">=</span> <span class="n">task_id</span> <span class="o">+</span> <span class="s2">"-"</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">uuid</span><span class="o">.</span><span class="n">uuid1</span><span class="p">())[:</span><span class="mi">8</span><span class="p">]</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">name</span> <span class="o">=</span> <span class="n">task_id</span>
+ <span class="n">name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_dataflow_job_name</span><span class="p">(</span><span class="n">task_id</span><span class="p">,</span> <span class="n">append_job_name</span><span class="p">)</span>
<span class="n">variables</span><span class="p">[</span><span class="s1">'jobName'</span><span class="p">]</span> <span class="o">=</span> <span class="n">name</span>
<span class="k">def</span> <span class="nf">label_formatter</span><span class="p">(</span><span class="n">labels_dict</span><span class="p">):</span>
@@ -368,19 +384,13 @@
<span class="k">def</span> <span class="nf">start_template_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">parameters</span><span class="p">,</span> <span class="n">dataflow_template</span><span class="p">,</span>
<span class="n">append_job_name</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
- <span class="k">if</span> <span class="n">append_job_name</span><span class="p">:</span>
- <span class="n">name</span> <span class="o">=</span> <span class="n">task_id</span> <span class="o">+</span> <span class="s2">"-"</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">uuid</span><span class="o">.</span><span class="n">uuid1</span><span class="p">())[:</span><span class="mi">8</span><span class="p">]</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">name</span> <span class="o">=</span> <span class="n">task_id</span>
+ <span class="n">name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_dataflow_job_name</span><span class="p">(</span><span class="n">task_id</span><span class="p">,</span> <span class="n">append_job_name</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_start_template_dataflow</span><span class="p">(</span>
<span class="n">name</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">parameters</span><span class="p">,</span> <span class="n">dataflow_template</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">start_python_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">dataflow</span><span class="p">,</span> <span class="n">py_options</span><span class="p">,</span>
<span class="n">append_job_name</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
- <span class="k">if</span> <span class="n">append_job_name</span><span class="p">:</span>
- <span class="n">name</span> <span class="o">=</span> <span class="n">task_id</span> <span class="o">+</span> <span class="s2">"-"</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">uuid</span><span class="o">.</span><span class="n">uuid1</span><span class="p">())[:</span><span class="mi">8</span><span class="p">]</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">name</span> <span class="o">=</span> <span class="n">task_id</span>
+ <span class="n">name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_dataflow_job_name</span><span class="p">(</span><span class="n">task_id</span><span class="p">,</span> <span class="n">append_job_name</span><span class="p">)</span>
<span class="n">variables</span><span class="p">[</span><span class="s1">'job_name'</span><span class="p">]</span> <span class="o">=</span> <span class="n">name</span>
<span class="k">def</span> <span class="nf">label_formatter</span><span class="p">(</span><span class="n">labels_dict</span><span class="p">):</span>
@@ -390,7 +400,25 @@
<span class="p">[</span><span class="s2">"python"</span><span class="p">]</span> <span class="o">+</span> <span class="n">py_options</span> <span class="o">+</span> <span class="p">[</span><span class="n">dataflow</span><span class="p">],</span>
<span class="n">label_formatter</span><span class="p">)</span>
- <span class="k">def</span> <span class="nf">_build_cmd</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">label_formatter</span><span class="p">):</span>
+ <span class="nd">@staticmethod</span>
+ <span class="k">def</span> <span class="nf">_build_dataflow_job_name</span><span class="p">(</span><span class="n">task_id</span><span class="p">,</span> <span class="n">append_job_name</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
+ <span class="n">task_id</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">task_id</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'_'</span><span class="p">,</span> <span class="s1">'-'</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="sa">r</span><span class="s2">"^[a-z]([-a-z0-9]*[a-z0-9])?$"</span><span class="p">,</span> <span class="n">task_id</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+ <span class="s1">'Invalid job_name (</span><span class="si">{}</span><span class="s1">); the name must consist of'</span>
+ <span class="s1">'only the characters [-a-z0-9], starting with a '</span>
+ <span class="s1">'letter and ending with a letter or number '</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">task_id</span><span class="p">))</span>
+
+ <span class="k">if</span> <span class="n">append_job_name</span><span class="p">:</span>
+ <span class="n">job_name</span> <span class="o">=</span> <span class="n">task_id</span> <span class="o">+</span> <span class="s2">"-"</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">uuid</span><span class="o">.</span><span class="n">uuid1</span><span class="p">())[:</span><span class="mi">8</span><span class="p">]</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">job_name</span> <span class="o">=</span> <span class="n">task_id</span>
+
+ <span class="k">return</span> <span class="n">job_name</span>
+
+ <span class="nd">@staticmethod</span>
+ <span class="k">def</span> <span class="nf">_build_cmd</span><span class="p">(</span><span class="n">task_id</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">label_formatter</span><span class="p">):</span>
<span class="n">command</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"--runner=DataflowRunner"</span><span class="p">]</span>
<span class="k">if</span> <span class="n">variables</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">for</span> <span class="n">attr</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">variables</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
@@ -402,7 +430,8 @@
<span class="n">command</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"--"</span> <span class="o">+</span> <span class="n">attr</span> <span class="o">+</span> <span class="s2">"="</span> <span class="o">+</span> <span class="n">value</span><span class="p">)</span>
<span class="k">return</span> <span class="n">command</span>
- <span class="k">def</span> <span class="nf">_start_template_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">parameters</span><span class="p">,</span> <span class="n">dataflow_template</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">_start_template_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">variables</span><span class="p">,</span> <span class="n">parameters</span><span class="p">,</span>
+ <span class="n">dataflow_template</span><span class="p">):</span>
<span class="c1"># Builds RuntimeEnvironment from variables dictionary</span>
<span class="c1"># https://cloud.google.com/dataflow/docs/reference/rest/v1b3/RuntimeEnvironment</span>
<span class="n">environment</span> <span class="o">=</span> <span class="p">{}</span>
@@ -414,9 +443,11 @@
<span class="s2">"parameters"</span><span class="p">:</span> <span class="n">parameters</span><span class="p">,</span>
<span class="s2">"environment"</span><span class="p">:</span> <span class="n">environment</span><span class="p">}</span>
<span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
- <span class="n">request</span> <span class="o">=</span> <span class="n">service</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">templates</span><span class="p">()</span><span class="o">.</span><span class="n">launch</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">variables</span><span class="p">[</span><span class="s1">'project'</span><span c [...]
- <span class="n">gcsPath</span><span class="o">=</span><span class="n">dataflow_template</span><span class="p">,</span>
- <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span>
+ <span class="n">request</span> <span class="o">=</span> <span class="n">service</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">templates</span><span class="p">()</span><span class="o">.</span><span class="n">launch</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="n">variables</span><span class="p">[</span><span class="s1">'project'</span><span class="p">],</span>
+ <span class="n">gcsPath</span><span class="o">=</span><span class="n">dataflow_template</span><span class="p">,</span>
+ <span class="n">body</span><span class="o">=</span><span class="n">body</span>
+ <span class="p">)</span>
<span class="n">response</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
<span class="n">variables</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_set_variables</span><span class="p">(</span><span class="n">variables</span><span class="p">)</span>
<span class="n">_DataflowJob</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">(),</span> <span class="n">variables</span><span class="p">[</span><span class="s1">'project'</span><span class="p">],</span> <span class="n">name</span><span class="p">,</span> <span class="n">variables</span><span class="p">[</span><span class="s1">'region'</span><span class="p">],</span>
@@ -452,20 +483,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/gcp_dataproc_hook.html b/_modules/airflow/contrib/hooks/gcp_dataproc_hook.html
index d22dfa5..2f062a9 100644
--- a/_modules/airflow/contrib/hooks/gcp_dataproc_hook.html
+++ b/_modules/airflow/contrib/hooks/gcp_dataproc_hook.html
@@ -429,20 +429,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/gcp_function_hook.html b/_modules/airflow/contrib/hooks/gcp_function_hook.html
new file mode 100644
index 0000000..366b837
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/gcp_function_hook.html
@@ -0,0 +1,406 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.gcp_function_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.gcp_function_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.gcp_function_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">time</span>
+<span class="kn">import</span> <span class="nn">requests</span>
+<span class="kn">from</span> <span class="nn">googleapiclient.discovery</span> <span class="k">import</span> <span class="n">build</span>
+
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="k">import</span> <span class="n">GoogleCloudBaseHook</span>
+
+<span class="c1"># Number of retries - used by googleapiclient method calls to perform retries</span>
+<span class="c1"># For requests that are "retriable"</span>
+<span class="n">NUM_RETRIES</span> <span class="o">=</span> <span class="mi">5</span>
+
+<span class="c1"># Time to sleep between active checks of the operation results</span>
+<span class="n">TIME_TO_SLEEP_IN_SECONDS</span> <span class="o">=</span> <span class="mi">1</span>
+
+
+<span class="c1"># noinspection PyAbstractClass</span>
+<div class="viewcode-block" id="GcfHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_function_hook.GcfHook">[docs]</a><span class="k">class</span> <span class="nc">GcfHook</span><span class="p">(</span><span class="n">GoogleCloudBaseHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Hook for the Google Cloud Functions APIs.</span>
+<span class="sd"> """</span>
+ <span class="n">_conn</span> <span class="o">=</span> <span class="kc">None</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">GcfHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">api_version</span> <span class="o">=</span> <span class="n">api_version</span>
+
+<div class="viewcode-block" id="GcfHook.get_conn"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_function_hook.GcfHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Retrieves the connection to Cloud Functions.</span>
+
+<span class="sd"> :return: Google Cloud Function services object</span>
+<span class="sd"> :rtype: dict</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_conn</span><span class="p">:</span>
+ <span class="n">http_authorized</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_conn</span> <span class="o">=</span> <span class="n">build</span><span class="p">(</span><span class="s1">'cloudfunctions'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">api_version</span><span class="p">,</span>
+ <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">,</span> <span class="n">cache_discovery</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_conn</span></div>
+
+<div class="viewcode-block" id="GcfHook.get_function"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_function_hook.GcfHook.get_function">[docs]</a> <span class="k">def</span> <span class="nf">get_function</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns the Cloud Function with the given name.</span>
+
+<span class="sd"> :param name: name of the function</span>
+<span class="sd"> :type name: str</span>
+<span class="sd"> :return: a CloudFunction object representing the function</span>
+<span class="sd"> :rtype: dict</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">locations</span><span class="p">()</span><span class="o">.</span><span class="n">functions</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="n">name</span><span class="o">=</span><span class="n">name</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">num_retries</span><span class="o">=</span><span class="n">NUM_RETRIES</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="GcfHook.list_functions"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_function_hook.GcfHook.list_functions">[docs]</a> <span class="k">def</span> <span class="nf">list_functions</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">full_location</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Lists all Cloud Functions created in the location.</span>
+
+<span class="sd"> :param full_location: full location including the project in the form of</span>
+<span class="sd"> of /projects/<PROJECT>/location/<LOCATION></span>
+<span class="sd"> :type full_location: str</span>
+<span class="sd"> :return: array of CloudFunction objects - representing functions in the location</span>
+<span class="sd"> :rtype: [dict]</span>
+<span class="sd"> """</span>
+ <span class="n">list_response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">locations</span><span class="p">()</span><span class="o">.</span><span class="n">functions</span><span class="p">()</span><span class="o">.</span><span class="n">list</span><span class="p">(</span>
+ <span class="n">parent</span><span class="o">=</span><span class="n">full_location</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">num_retries</span><span class="o">=</span><span class="n">NUM_RETRIES</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">list_response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"functions"</span><span class="p">,</span> <span class="p">[])</span></div>
+
+<div class="viewcode-block" id="GcfHook.create_new_function"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_function_hook.GcfHook.create_new_function">[docs]</a> <span class="k">def</span> <span class="nf">create_new_function</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">full_location</span><span class="p">,</span> <span class="n">body</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Creates a new function in Cloud Function in the location specified in the body.</span>
+
+<span class="sd"> :param full_location: full location including the project in the form of</span>
+<span class="sd"> of /projects/<PROJECT>/location/<LOCATION></span>
+<span class="sd"> :type full_location: str</span>
+<span class="sd"> :param body: body required by the Cloud Functions insert API</span>
+<span class="sd"> :type body: dict</span>
+<span class="sd"> :return: response returned by the operation</span>
+<span class="sd"> :rtype: dict</span>
+<span class="sd"> """</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">locations</span><span class="p">()</span><span class="o">.</span><span class="n">functions</span><span class="p">()</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
+ <span class="n">location</span><span class="o">=</span><span class="n">full_location</span><span class="p">,</span>
+ <span class="n">body</span><span class="o">=</span><span class="n">body</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">num_retries</span><span class="o">=</span><span class="n">NUM_RETRIES</span><span class="p">)</span>
+ <span class="n">operation_name</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s2">"name"</span><span class="p">]</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_operation_to_complete</span><span class="p">(</span><span class="n">operation_name</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="GcfHook.update_function"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_function_hook.GcfHook.update_function">[docs]</a> <span class="k">def</span> <span class="nf">update_function</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">body</span><span class="p">,</span> <span class="n">update_mask</span><span class="p" [...]
+ <span class="sd">"""</span>
+<span class="sd"> Updates Cloud Functions according to the specified update mask.</span>
+
+<span class="sd"> :param name: name of the function</span>
+<span class="sd"> :type name: str</span>
+<span class="sd"> :param body: body required by the cloud function patch API</span>
+<span class="sd"> :type body: str</span>
+<span class="sd"> :param update_mask: update mask - array of fields that should be patched</span>
+<span class="sd"> :type update_mask: [str]</span>
+<span class="sd"> :return: response returned by the operation</span>
+<span class="sd"> :rtype: dict</span>
+<span class="sd"> """</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">locations</span><span class="p">()</span><span class="o">.</span><span class="n">functions</span><span class="p">()</span><span class="o">.</span><span class="n">patch</span><span class="p">(</span>
+ <span class="n">updateMask</span><span class="o">=</span><span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">update_mask</span><span class="p">),</span>
+ <span class="n">name</span><span class="o">=</span><span class="n">name</span><span class="p">,</span>
+ <span class="n">body</span><span class="o">=</span><span class="n">body</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">num_retries</span><span class="o">=</span><span class="n">NUM_RETRIES</span><span class="p">)</span>
+ <span class="n">operation_name</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s2">"name"</span><span class="p">]</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_operation_to_complete</span><span class="p">(</span><span class="n">operation_name</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="GcfHook.upload_function_zip"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_function_hook.GcfHook.upload_function_zip">[docs]</a> <span class="k">def</span> <span class="nf">upload_function_zip</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">parent</span><span class="p">,</span> <span class="n">zip_path</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Uploads zip file with sources.</span>
+
+<span class="sd"> :param parent: Google Cloud Platform project id and region where zip file should</span>
+<span class="sd"> be uploaded in the form of /projects/<PROJECT>/location/<LOCATION></span>
+<span class="sd"> :type parent: str</span>
+<span class="sd"> :param zip_path: path of the valid .zip file to upload</span>
+<span class="sd"> :type zip_path: str</span>
+<span class="sd"> :return: Upload URL that was returned by generateUploadUrl method</span>
+<span class="sd"> """</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">locations</span><span class="p">()</span><span class="o">.</span><span class="n">functions</span><span class="p">()</span><span class="o">.</span><span class="n">generateUploadUrl</span><span class=" [...]
+ <span class="n">parent</span><span class="o">=</span><span class="n">parent</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">num_retries</span><span class="o">=</span><span class="n">NUM_RETRIES</span><span class="p">)</span>
+ <span class="n">upload_url</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'uploadUrl'</span><span class="p">)</span>
+ <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">zip_path</span><span class="p">,</span> <span class="s1">'rb'</span><span class="p">)</span> <span class="k">as</span> <span class="n">fp</span><span class="p">:</span>
+ <span class="n">requests</span><span class="o">.</span><span class="n">put</span><span class="p">(</span>
+ <span class="n">url</span><span class="o">=</span><span class="n">upload_url</span><span class="p">,</span>
+ <span class="n">data</span><span class="o">=</span><span class="n">fp</span><span class="o">.</span><span class="n">read</span><span class="p">(),</span>
+ <span class="c1"># Those two headers needs to be specified according to:</span>
+ <span class="c1"># https://cloud.google.com/functions/docs/reference/rest/v1/projects.locations.functions/generateUploadUrl</span>
+ <span class="c1"># nopep8</span>
+ <span class="n">headers</span><span class="o">=</span><span class="p">{</span>
+ <span class="s1">'Content-type'</span><span class="p">:</span> <span class="s1">'application/zip'</span><span class="p">,</span>
+ <span class="s1">'x-goog-content-length-range'</span><span class="p">:</span> <span class="s1">'0,104857600'</span><span class="p">,</span>
+ <span class="p">}</span>
+ <span class="p">)</span>
+ <span class="k">return</span> <span class="n">upload_url</span></div>
+
+<div class="viewcode-block" id="GcfHook.delete_function"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_function_hook.GcfHook.delete_function">[docs]</a> <span class="k">def</span> <span class="nf">delete_function</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Deletes the specified Cloud Function.</span>
+
+<span class="sd"> :param name: name of the function</span>
+<span class="sd"> :type name: str</span>
+<span class="sd"> :return: response returned by the operation</span>
+<span class="sd"> :rtype: dict</span>
+<span class="sd"> """</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">locations</span><span class="p">()</span><span class="o">.</span><span class="n">functions</span><span class="p">()</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span>
+ <span class="n">name</span><span class="o">=</span><span class="n">name</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">num_retries</span><span class="o">=</span><span class="n">NUM_RETRIES</span><span class="p">)</span>
+ <span class="n">operation_name</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s2">"name"</span><span class="p">]</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_operation_to_complete</span><span class="p">(</span><span class="n">operation_name</span><span class="p">)</span></div>
+
+ <span class="k">def</span> <span class="nf">_wait_for_operation_to_complete</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Waits for the named operation to complete - checks status of the</span>
+<span class="sd"> asynchronous call.</span>
+
+<span class="sd"> :param operation_name: name of the operation</span>
+<span class="sd"> :type operation_name: str</span>
+<span class="sd"> :return: response returned by the operation</span>
+<span class="sd"> :rtype: dict</span>
+<span class="sd"> :exception: AirflowException in case error is returned</span>
+<span class="sd"> """</span>
+ <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+ <span class="n">operation_response</span> <span class="o">=</span> <span class="n">service</span><span class="o">.</span><span class="n">operations</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="n">name</span><span class="o">=</span><span class="n">operation_name</span><span class="p">,</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">num_retries</span><span class="o">=</span><span class="n">NUM_RETRIES</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">operation_response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"done"</span><span class="p">):</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="n">operation_response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"response"</span><span class="p">)</span>
+ <span class="n">error</span> <span class="o">=</span> <span class="n">operation_response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"error"</span><span class="p">)</span>
+ <span class="c1"># Note, according to documentation always either response or error is</span>
+ <span class="c1"># set when "done" == True</span>
+ <span class="k">if</span> <span class="n">error</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">error</span><span class="p">))</span>
+ <span class="k">return</span> <span class="n">response</span>
+ <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="n">TIME_TO_SLEEP_IN_SECONDS</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/gcp_mlengine_hook.html b/_modules/airflow/contrib/hooks/gcp_mlengine_hook.html
index b961738..f3b1782 100644
--- a/_modules/airflow/contrib/hooks/gcp_mlengine_hook.html
+++ b/_modules/airflow/contrib/hooks/gcp_mlengine_hook.html
@@ -313,7 +313,8 @@
<span class="sd"> apiclient.errors.HttpError: if HTTP error is returned when getting</span>
<span class="sd"> the job</span>
<span class="sd"> """</span>
- <span class="k">assert</span> <span class="n">interval</span> <span class="o">></span> <span class="mi">0</span>
+ <span class="k">if</span> <span class="n">interval</span> <span class="o"><=</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Interval must be > 0"</span><span class="p">)</span>
<span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
<span class="n">job</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_job</span><span class="p">(</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job_id</span><span class="p">)</span>
<span class="k">if</span> <span class="n">job</span><span class="p">[</span><span class="s1">'state'</span><span class="p">]</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'SUCCEEDED'</span><span class="p">,</span> <span class="s1">'FAILED'</span><span class="p">,</span> <span class="s1">'CANCELLED'</span><span class="p">]:</span>
@@ -403,7 +404,9 @@
<span class="sd">"""</span>
<span class="sd"> Create a Model. Blocks until finished.</span>
<span class="sd"> """</span>
- <span class="k">assert</span> <span class="n">model</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">model</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="s1">''</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">model</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Model name must be provided and "</span>
+ <span class="s2">"could not be an empty string"</span><span class="p">)</span>
<span class="n">project</span> <span class="o">=</span> <span class="s1">'projects/</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">project_id</span><span class="p">)</span>
<span class="n">request</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mlengine</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">models</span><span class="p">()</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
@@ -414,7 +417,9 @@
<span class="sd">"""</span>
<span class="sd"> Gets a Model. Blocks until finished.</span>
<span class="sd"> """</span>
- <span class="k">assert</span> <span class="n">model_name</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">model_name</span> <span class="ow">is</span> <span class="ow">not</span> <span class="s1">''</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">model_name</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Model name must be provided and "</span>
+ <span class="s2">"it could not be an empty string"</span><span class="p">)</span>
<span class="n">full_model_name</span> <span class="o">=</span> <span class="s1">'projects/</span><span class="si">{}</span><span class="s1">/models/</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="n">project_id</span><span class="p">,</span> <span class="n">model_name</span><span class="p">)</span>
<span class="n">request</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mlengine</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">models</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="n">full_model_name</span><span class="p">)</span>
@@ -455,20 +460,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/gcp_pubsub_hook.html b/_modules/airflow/contrib/hooks/gcp_pubsub_hook.html
index 1cb0cac..231392a 100644
--- a/_modules/airflow/contrib/hooks/gcp_pubsub_hook.html
+++ b/_modules/airflow/contrib/hooks/gcp_pubsub_hook.html
@@ -474,20 +474,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/gcp_sql_hook.html b/_modules/airflow/contrib/hooks/gcp_sql_hook.html
new file mode 100644
index 0000000..9428e74
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/gcp_sql_hook.html
@@ -0,0 +1,474 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.gcp_sql_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.gcp_sql_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.gcp_sql_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">time</span>
+<span class="kn">from</span> <span class="nn">googleapiclient.discovery</span> <span class="k">import</span> <span class="n">build</span>
+
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="k">import</span> <span class="n">GoogleCloudBaseHook</span>
+
+<span class="c1"># Number of retries - used by googleapiclient method calls to perform retries</span>
+<span class="c1"># For requests that are "retriable"</span>
+<span class="n">NUM_RETRIES</span> <span class="o">=</span> <span class="mi">5</span>
+
+<span class="c1"># Time to sleep between active checks of the operation results</span>
+<span class="n">TIME_TO_SLEEP_IN_SECONDS</span> <span class="o">=</span> <span class="mi">1</span>
+
+
+<span class="k">class</span> <span class="nc">CloudSqlOperationStatus</span><span class="p">:</span>
+ <span class="n">PENDING</span> <span class="o">=</span> <span class="s2">"PENDING"</span>
+ <span class="n">RUNNING</span> <span class="o">=</span> <span class="s2">"RUNNING"</span>
+ <span class="n">DONE</span> <span class="o">=</span> <span class="s2">"DONE"</span>
+ <span class="n">UNKNOWN</span> <span class="o">=</span> <span class="s2">"UNKNOWN"</span>
+
+
+<span class="c1"># noinspection PyAbstractClass</span>
+<div class="viewcode-block" id="CloudSqlHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook">[docs]</a><span class="k">class</span> <span class="nc">CloudSqlHook</span><span class="p">(</span><span class="n">GoogleCloudBaseHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Hook for Google Cloud SQL APIs.</span>
+<span class="sd"> """</span>
+ <span class="n">_conn</span> <span class="o">=</span> <span class="kc">None</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">CloudSqlHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">api_version</span> <span class="o">=</span> <span class="n">api_version</span>
+
+<div class="viewcode-block" id="CloudSqlHook.get_conn"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Retrieves connection to Cloud SQL.</span>
+
+<span class="sd"> :return: Google Cloud SQL services object.</span>
+<span class="sd"> :rtype: dict</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_conn</span><span class="p">:</span>
+ <span class="n">http_authorized</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_conn</span> <span class="o">=</span> <span class="n">build</span><span class="p">(</span><span class="s1">'sqladmin'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">api_version</span><span class="p">,</span>
+ <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">,</span> <span class="n">cache_discovery</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_conn</span></div>
+
+<div class="viewcode-block" id="CloudSqlHook.get_instance"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_instance">[docs]</a> <span class="k">def</span> <span class="nf">get_instance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">instance</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Retrieves a resource containing information about a Cloud SQL instance.</span>
+
+<span class="sd"> :param project_id: Project ID of the project that contains the instance.</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :param instance: Database instance ID. This does not include the project ID.</span>
+<span class="sd"> :type instance: str</span>
+<span class="sd"> :return: A Cloud SQL instance resource.</span>
+<span class="sd"> :rtype: dict</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">instances</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="n">project</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">instance</span><span class="o">=</span><span class="n">instance</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">num_retries</span><span class="o">=</span><span class="n">NUM_RETRIES</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="CloudSqlHook.create_instance"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.create_instance">[docs]</a> <span class="k">def</span> <span class="nf">create_instance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Creates a new Cloud SQL instance.</span>
+
+<span class="sd"> :param project_id: Project ID of the project to which the newly created</span>
+<span class="sd"> Cloud SQL instances should belong.</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :param body: Body required by the Cloud SQL insert API, as described in</span>
+<span class="sd"> https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/insert#request-body</span>
+<span class="sd"> :type body: dict</span>
+<span class="sd"> :return: True if the operation succeeded, raises an error otherwise</span>
+<span class="sd"> :rtype: bool</span>
+<span class="sd"> """</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">instances</span><span class="p">()</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span>
+ <span class="n">project</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">body</span><span class="o">=</span><span class="n">body</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">num_retries</span><span class="o">=</span><span class="n">NUM_RETRIES</span><span class="p">)</span>
+ <span class="n">operation_name</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s2">"name"</span><span class="p">]</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_operation_to_complete</span><span class="p">(</span><span class="n">project_id</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="CloudSqlHook.patch_instance"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.patch_instance">[docs]</a> <span class="k">def</span> <span class="nf">patch_instance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="p">,</span> <span class="n">instance</span><span clas [...]
+ <span class="sd">"""</span>
+<span class="sd"> Updates settings of a Cloud SQL instance.</span>
+
+<span class="sd"> Caution: This is not a partial update, so you must include values for</span>
+<span class="sd"> all the settings that you want to retain.</span>
+
+<span class="sd"> :param project_id: Project ID of the project that contains the instance.</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :param body: Body required by the Cloud SQL patch API, as described in</span>
+<span class="sd"> https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/patch#request-body</span>
+<span class="sd"> :type body: dict</span>
+<span class="sd"> :param instance: Cloud SQL instance ID. This does not include the project ID.</span>
+<span class="sd"> :type instance: str</span>
+<span class="sd"> :return: True if the operation succeeded, raises an error otherwise</span>
+<span class="sd"> :rtype: bool</span>
+<span class="sd"> """</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">instances</span><span class="p">()</span><span class="o">.</span><span class="n">patch</span><span class="p">(</span>
+ <span class="n">project</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">instance</span><span class="o">=</span><span class="n">instance</span><span class="p">,</span>
+ <span class="n">body</span><span class="o">=</span><span class="n">body</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">num_retries</span><span class="o">=</span><span class="n">NUM_RETRIES</span><span class="p">)</span>
+ <span class="n">operation_name</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s2">"name"</span><span class="p">]</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_operation_to_complete</span><span class="p">(</span><span class="n">project_id</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="CloudSqlHook.delete_instance"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.delete_instance">[docs]</a> <span class="k">def</span> <span class="nf">delete_instance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">instance</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Deletes a Cloud SQL instance.</span>
+
+<span class="sd"> :param project_id: Project ID of the project that contains the instance.</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :param instance: Cloud SQL instance ID. This does not include the project ID.</span>
+<span class="sd"> :type instance: str</span>
+<span class="sd"> :return: True if the operation succeeded, raises an error otherwise</span>
+<span class="sd"> :rtype: bool</span>
+<span class="sd"> """</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">instances</span><span class="p">()</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span>
+ <span class="n">project</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">instance</span><span class="o">=</span><span class="n">instance</span><span class="p">,</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">num_retries</span><span class="o">=</span><span class="n">NUM_RETRIES</span><span class="p">)</span>
+ <span class="n">operation_name</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s2">"name"</span><span class="p">]</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_operation_to_complete</span><span class="p">(</span><span class="n">project_id</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="CloudSqlHook.get_database"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_database">[docs]</a> <span class="k">def</span> <span class="nf">get_database</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">instance</span><span class="p">,</span> <span class="n">database</span><span class= [...]
+ <span class="sd">"""</span>
+<span class="sd"> Retrieves a database resource from a Cloud SQL instance.</span>
+
+<span class="sd"> :param project_id: Project ID of the project that contains the instance.</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :param instance: Database instance ID. This does not include the project ID.</span>
+<span class="sd"> :type instance: str</span>
+<span class="sd"> :param database: Name of the database in the instance.</span>
+<span class="sd"> :type database: str</span>
+<span class="sd"> :return: A Cloud SQL database resource, as described in</span>
+<span class="sd"> https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/databases#resource</span>
+<span class="sd"> :rtype: dict</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">databases</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="n">project</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">instance</span><span class="o">=</span><span class="n">instance</span><span class="p">,</span>
+ <span class="n">database</span><span class="o">=</span><span class="n">database</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">num_retries</span><span class="o">=</span><span class="n">NUM_RETRIES</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="CloudSqlHook.create_database"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.create_database">[docs]</a> <span class="k">def</span> <span class="nf">create_database</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project</span><span class="p">,</span> <span class="n">instance</span><span class="p">,</span> <span class="n">body</span><span clas [...]
+ <span class="sd">"""</span>
+<span class="sd"> Creates a new database inside a Cloud SQL instance.</span>
+
+<span class="sd"> :param project: Project ID of the project that contains the instance.</span>
+<span class="sd"> :type project: str</span>
+<span class="sd"> :param instance: Database instance ID. This does not include the project ID.</span>
+<span class="sd"> :type instance: str</span>
+<span class="sd"> :param body: The request body, as described in</span>
+<span class="sd"> https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/databases/insert#request-body</span>
+<span class="sd"> :type body: dict</span>
+<span class="sd"> :return: True if the operation succeeded, raises an error otherwise</span>
+<span class="sd"> :rtype: bool</span>
+<span class="sd"> """</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">databases</span><span class="p">()</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span>
+ <span class="n">project</span><span class="o">=</span><span class="n">project</span><span class="p">,</span>
+ <span class="n">instance</span><span class="o">=</span><span class="n">instance</span><span class="p">,</span>
+ <span class="n">body</span><span class="o">=</span><span class="n">body</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">num_retries</span><span class="o">=</span><span class="n">NUM_RETRIES</span><span class="p">)</span>
+ <span class="n">operation_name</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s2">"name"</span><span class="p">]</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_operation_to_complete</span><span class="p">(</span><span class="n">project</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="CloudSqlHook.patch_database"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.patch_database">[docs]</a> <span class="k">def</span> <span class="nf">patch_database</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project</span><span class="p">,</span> <span class="n">instance</span><span class="p">,</span> <span class="n">database</span><span cla [...]
+ <span class="sd">"""</span>
+<span class="sd"> Updates a database resource inside a Cloud SQL instance.</span>
+<span class="sd"> This method supports patch semantics.</span>
+<span class="sd"> See: https://cloud.google.com/sql/docs/mysql/admin-api/how-tos/performance#patch</span>
+
+<span class="sd"> :param project: Project ID of the project that contains the instance.</span>
+<span class="sd"> :type project: str</span>
+<span class="sd"> :param instance: Database instance ID. This does not include the project ID.</span>
+<span class="sd"> :type instance: str</span>
+<span class="sd"> :param database: Name of the database to be updated in the instance.</span>
+<span class="sd"> :type database: str</span>
+<span class="sd"> :param body: The request body, as described in</span>
+<span class="sd"> https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/databases/insert#request-body</span>
+<span class="sd"> :type body: dict</span>
+<span class="sd"> :return: True if the operation succeeded, raises an error otherwise</span>
+<span class="sd"> :rtype: bool</span>
+<span class="sd"> """</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">databases</span><span class="p">()</span><span class="o">.</span><span class="n">patch</span><span class="p">(</span>
+ <span class="n">project</span><span class="o">=</span><span class="n">project</span><span class="p">,</span>
+ <span class="n">instance</span><span class="o">=</span><span class="n">instance</span><span class="p">,</span>
+ <span class="n">database</span><span class="o">=</span><span class="n">database</span><span class="p">,</span>
+ <span class="n">body</span><span class="o">=</span><span class="n">body</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">num_retries</span><span class="o">=</span><span class="n">NUM_RETRIES</span><span class="p">)</span>
+ <span class="n">operation_name</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s2">"name"</span><span class="p">]</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_operation_to_complete</span><span class="p">(</span><span class="n">project</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="CloudSqlHook.delete_database"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.delete_database">[docs]</a> <span class="k">def</span> <span class="nf">delete_database</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project</span><span class="p">,</span> <span class="n">instance</span><span class="p">,</span> <span class="n">database</span><span [...]
+ <span class="sd">"""</span>
+<span class="sd"> Deletes a database from a Cloud SQL instance.</span>
+
+<span class="sd"> :param project: Project ID of the project that contains the instance.</span>
+<span class="sd"> :type project: str</span>
+<span class="sd"> :param instance: Database instance ID. This does not include the project ID.</span>
+<span class="sd"> :type instance: str</span>
+<span class="sd"> :param database: Name of the database to be deleted in the instance.</span>
+<span class="sd"> :type database: str</span>
+<span class="sd"> :return: True if the operation succeeded, raises an error otherwise</span>
+<span class="sd"> :rtype: bool</span>
+<span class="sd"> """</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">databases</span><span class="p">()</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span>
+ <span class="n">project</span><span class="o">=</span><span class="n">project</span><span class="p">,</span>
+ <span class="n">instance</span><span class="o">=</span><span class="n">instance</span><span class="p">,</span>
+ <span class="n">database</span><span class="o">=</span><span class="n">database</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">num_retries</span><span class="o">=</span><span class="n">NUM_RETRIES</span><span class="p">)</span>
+ <span class="n">operation_name</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s2">"name"</span><span class="p">]</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_operation_to_complete</span><span class="p">(</span><span class="n">project</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">)</span></div>
+
+ <span class="k">def</span> <span class="nf">_wait_for_operation_to_complete</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Waits for the named operation to complete - checks status of the</span>
+<span class="sd"> asynchronous call.</span>
+
+<span class="sd"> :param project_id: Project ID of the project that contains the instance.</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :param operation_name: name of the operation</span>
+<span class="sd"> :type operation_name: str</span>
+<span class="sd"> :return: response returned by the operation</span>
+<span class="sd"> :rtype: dict</span>
+<span class="sd"> """</span>
+ <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+ <span class="n">operation_response</span> <span class="o">=</span> <span class="n">service</span><span class="o">.</span><span class="n">operations</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="n">project</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">operation</span><span class="o">=</span><span class="n">operation_name</span><span class="p">,</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">num_retries</span><span class="o">=</span><span class="n">NUM_RETRIES</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">operation_response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"status"</span><span class="p">)</span> <span class="o">==</span> <span class="n">CloudSqlOperationStatus</span><span class="o">.</span><span class="n">DONE</span><span class="p">:</span>
+ <span class="n">error</span> <span class="o">=</span> <span class="n">operation_response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"error"</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">error</span><span class="p">:</span>
+ <span class="c1"># Extracting the errors list as string and trimming square braces</span>
+ <span class="n">error_msg</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"errors"</span><span class="p">))[</span><span class="mi">1</span><span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="n">error_msg</span><span class="p">)</span>
+ <span class="c1"># No meaningful info to return from the response in case of success</span>
+ <span class="k">return</span> <span class="kc">True</span>
+ <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="n">TIME_TO_SLEEP_IN_SECONDS</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/gcs_hook.html b/_modules/airflow/contrib/hooks/gcs_hook.html
index c4896fb..a72ac0d 100644
--- a/_modules/airflow/contrib/hooks/gcs_hook.html
+++ b/_modules/airflow/contrib/hooks/gcs_hook.html
@@ -185,7 +185,10 @@
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="k">import</span> <span class="n">GoogleCloudBaseHook</span>
<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">import</span> <span class="nn">gzip</span> <span class="k">as</span> <span class="nn">gz</span>
+<span class="kn">import</span> <span class="nn">shutil</span>
<span class="kn">import</span> <span class="nn">re</span>
+<span class="kn">import</span> <span class="nn">os</span>
<div class="viewcode-block" id="GoogleCloudStorageHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook">[docs]</a><span class="k">class</span> <span class="nc">GoogleCloudStorageHook</span><span class="p">(</span><span class="n">GoogleCloudBaseHook</span><span class="p">):</span>
@@ -332,7 +335,8 @@
<span class="k">return</span> <span class="n">downloaded_file_bytes</span></div>
<span class="c1"># pylint:disable=redefined-builtin</span>
-<div class="viewcode-block" id="GoogleCloudStorageHook.upload"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.upload">[docs]</a> <span class="k">def</span> <span class="nf">upload</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</ [...]
+<div class="viewcode-block" id="GoogleCloudStorageHook.upload"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.upload">[docs]</a> <span class="k">def</span> <span class="nf">upload</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</span>
+ <span class="n">mime_type</span><span class="o">=</span><span class="s1">'application/octet-stream'</span><span class="p">,</span> <span class="n">gzip</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Uploads a local file to Google Cloud Storage.</span>
@@ -343,15 +347,31 @@
<span class="sd"> :param filename: The local file path to the file to be uploaded.</span>
<span class="sd"> :type filename: string</span>
<span class="sd"> :param mime_type: The MIME type to set when uploading the file.</span>
-<span class="sd"> :type mime_type: string</span>
+<span class="sd"> :type mime_type: str</span>
+<span class="sd"> :param gzip: Option to compress file for upload</span>
+<span class="sd"> :type gzip: bool</span>
<span class="sd"> """</span>
<span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+ <span class="k">if</span> <span class="n">gzip</span><span class="p">:</span>
+ <span class="n">filename_gz</span> <span class="o">=</span> <span class="n">filename</span> <span class="o">+</span> <span class="s1">'.gz'</span>
+
+ <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s1">'rb'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f_in</span><span class="p">:</span>
+ <span class="k">with</span> <span class="n">gz</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">filename_gz</span><span class="p">,</span> <span class="s1">'wb'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f_out</span><span class="p">:</span>
+ <span class="n">shutil</span><span class="o">.</span><span class="n">copyfileobj</span><span class="p">(</span><span class="n">f_in</span><span class="p">,</span> <span class="n">f_out</span><span class="p">)</span>
+ <span class="n">filename</span> <span class="o">=</span> <span class="n">filename_gz</span>
+
<span class="n">media</span> <span class="o">=</span> <span class="n">MediaFileUpload</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="n">mime_type</span><span class="p">)</span>
+
<span class="k">try</span><span class="p">:</span>
<span class="n">service</span> \
<span class="o">.</span><span class="n">objects</span><span class="p">()</span> \
<span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="n">bucket</span><span class="o">=</span><span class="n">bucket</span><span class="p">,</span> <span class="n">name</span><span class="o">=</span><span class="nb">object</span><span class="p">,</span> <span class="n">media_body</span><span class="o">=</span><span class="n">media</span><span class="p">)</span> \
<span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+
+ <span class="c1"># Clean up gzip file</span>
+ <span class="k">if</span> <span class="n">gzip</span><span class="p">:</span>
+ <span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">filename</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">True</span>
<span class="k">except</span> <span class="n">errors</span><span class="o">.</span><span class="n">HttpError</span> <span class="k">as</span> <span class="n">ex</span><span class="p">:</span>
<span class="k">if</span> <span class="n">ex</span><span class="o">.</span><span class="n">resp</span><span class="p">[</span><span class="s1">'status'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'404'</span><span class="p">:</span>
@@ -638,15 +658,16 @@
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Creating Bucket: </span><span class="si">%s</span><span class="s1">; Location: </span><span class="si">%s</span><span class="s1">; Storage Class: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
<span class="n">bucket_name</span><span class="p">,</span> <span class="n">location</span><span class="p">,</span> <span class="n">storage_class</span><span class="p">)</span>
- <span class="k">assert</span> <span class="n">storage_class</span> <span class="ow">in</span> <span class="n">storage_classes</span><span class="p">,</span> \
- <span class="s1">'Invalid value (</span><span class="si">{}</span><span class="s1">) passed to storage_class. Value should be '</span> \
- <span class="s1">'one of </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">storage_class</span><span class="p">,</span> <span class="n">storage_classes</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">storage_class</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">storage_classes</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+ <span class="s1">'Invalid value (</span><span class="si">{}</span><span class="s1">) passed to storage_class. Value should be '</span>
+ <span class="s1">'one of </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">storage_class</span><span class="p">,</span> <span class="n">storage_classes</span><span class="p">))</span>
- <span class="k">assert</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="s1">'[a-zA-Z0-9]+'</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">[</span><span class="mi">0</span><span class="p">]),</span> \
- <span class="s1">'Bucket names must start with a number or letter.'</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="s1">'[a-zA-Z0-9]+'</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">[</span><span class="mi">0</span><span class="p">]):</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'Bucket names must start with a number or letter.'</span><span class="p">)</span>
- <span class="k">assert</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="s1">'[a-zA-Z0-9]+'</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]),</span> \
- <span class="s1">'Bucket names must end with a number or letter.'</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="s1">'[a-zA-Z0-9]+'</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]):</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'Bucket names must end with a number or letter.'</span><span class="p">)</span>
<span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
<span class="n">bucket_resource</span> <span class="o">=</span> <span class="p">{</span>
@@ -726,20 +747,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/imap_hook.html b/_modules/airflow/contrib/hooks/imap_hook.html
new file mode 100644
index 0000000..c9d9e48
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/imap_hook.html
@@ -0,0 +1,491 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.imap_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.imap_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.imap_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">email</span>
+<span class="kn">import</span> <span class="nn">imaplib</span>
+<span class="kn">import</span> <span class="nn">os</span>
+<span class="kn">import</span> <span class="nn">re</span>
+
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+
+
+<div class="viewcode-block" id="ImapHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.imap_hook.ImapHook">[docs]</a><span class="k">class</span> <span class="nc">ImapHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> This hook connects to a mail server by using the imap protocol.</span>
+
+<span class="sd"> :param imap_conn_id: The connection id that contains the information</span>
+<span class="sd"> used to authenticate the client.</span>
+<span class="sd"> The default value is 'imap_default'.</span>
+<span class="sd"> :type imap_conn_id: str</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">imap_conn_id</span><span class="o">=</span><span class="s1">'imap_default'</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">ImapHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">imap_conn_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">imap_conn_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mail_client</span> <span class="o">=</span> <span class="n">imaplib</span><span class="o">.</span><span class="n">IMAP4_SSL</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">__enter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mail_client</span><span class="o">.</span><span class="n">login</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span>
+
+ <span class="k">def</span> <span class="nf">__exit__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">exc_type</span><span class="p">,</span> <span class="n">exc_val</span><span class="p">,</span> <span class="n">exc_tb</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mail_client</span><span class="o">.</span><span class="n">logout</span><span class="p">()</span>
+
+<div class="viewcode-block" id="ImapHook.has_mail_attachment"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.imap_hook.ImapHook.has_mail_attachment">[docs]</a> <span class="k">def</span> <span class="nf">has_mail_attachment</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">mail_folder</span><span class="o">=</span><span class="s1">'INBOX'</span><span [...]
+ <span class="sd">"""</span>
+<span class="sd"> Checks the mail folder for mails containing attachments with the given name.</span>
+
+<span class="sd"> :param name: The name of the attachment that will be searched for.</span>
+<span class="sd"> :type name: str</span>
+<span class="sd"> :param mail_folder: The mail folder where to look at.</span>
+<span class="sd"> The default value is 'INBOX'.</span>
+<span class="sd"> :type mail_folder: str</span>
+<span class="sd"> :param check_regex: Checks the name for a regular expression.</span>
+<span class="sd"> The default value is False.</span>
+<span class="sd"> :type check_regex: bool</span>
+<span class="sd"> :returns: True if there is an attachment with the given name and False if not.</span>
+<span class="sd"> :rtype: bool</span>
+<span class="sd"> """</span>
+ <span class="n">mail_attachments</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_retrieve_mails_attachments_by_name</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">mail_folder</span><span class="p">,</span>
+ <span class="n">check_regex</span><span class="p">,</span>
+ <span class="n">latest_only</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="n">mail_attachments</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span></div>
+
+<div class="viewcode-block" id="ImapHook.retrieve_mail_attachments"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.imap_hook.ImapHook.retrieve_mail_attachments">[docs]</a> <span class="k">def</span> <span class="nf">retrieve_mail_attachments</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">mail_folder</span><span class="o">=</span><span class="s1">'INBO [...]
+ <span class="n">latest_only</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Retrieves mail's attachments in the mail folder by its name.</span>
+
+<span class="sd"> :param name: The name of the attachment that will be downloaded.</span>
+<span class="sd"> :type name: str</span>
+<span class="sd"> :param mail_folder: The mail folder where to look at.</span>
+<span class="sd"> The default value is 'INBOX'.</span>
+<span class="sd"> :type mail_folder: str</span>
+<span class="sd"> :param check_regex: Checks the name for a regular expression.</span>
+<span class="sd"> The default value is False.</span>
+<span class="sd"> :type check_regex: bool</span>
+<span class="sd"> :param latest_only: If set to True it will only retrieve</span>
+<span class="sd"> the first matched attachment.</span>
+<span class="sd"> The default value is False.</span>
+<span class="sd"> :type latest_only: bool</span>
+<span class="sd"> :returns: a list of tuple each containing the attachment filename and its payload.</span>
+<span class="sd"> :rtype: a list of tuple</span>
+<span class="sd"> """</span>
+ <span class="n">mail_attachments</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_retrieve_mails_attachments_by_name</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">mail_folder</span><span class="p">,</span>
+ <span class="n">check_regex</span><span class="p">,</span>
+ <span class="n">latest_only</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">mail_attachments</span></div>
+
+<div class="viewcode-block" id="ImapHook.download_mail_attachments"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.imap_hook.ImapHook.download_mail_attachments">[docs]</a> <span class="k">def</span> <span class="nf">download_mail_attachments</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">local_output_directory</span><span class="p">,</span> <span class="n [...]
+ <span class="n">check_regex</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">latest_only</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Downloads mail's attachments in the mail folder by its name</span>
+<span class="sd"> to the local directory.</span>
+
+<span class="sd"> :param name: The name of the attachment that will be downloaded.</span>
+<span class="sd"> :type name: str</span>
+<span class="sd"> :param local_output_directory: The output directory on the local machine</span>
+<span class="sd"> where the files will be downloaded to.</span>
+<span class="sd"> :type local_output_directory: str</span>
+<span class="sd"> :param mail_folder: The mail folder where to look at.</span>
+<span class="sd"> The default value is 'INBOX'.</span>
+<span class="sd"> :type mail_folder: str</span>
+<span class="sd"> :param check_regex: Checks the name for a regular expression.</span>
+<span class="sd"> The default value is False.</span>
+<span class="sd"> :type check_regex: bool</span>
+<span class="sd"> :param latest_only: If set to True it will only download</span>
+<span class="sd"> the first matched attachment.</span>
+<span class="sd"> The default value is False.</span>
+<span class="sd"> :type latest_only: bool</span>
+<span class="sd"> """</span>
+ <span class="n">mail_attachments</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_retrieve_mails_attachments_by_name</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">mail_folder</span><span class="p">,</span>
+ <span class="n">check_regex</span><span class="p">,</span> <span class="n">latest_only</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_create_files</span><span class="p">(</span><span class="n">mail_attachments</span><span class="p">,</span> <span class="n">local_output_directory</span><span class="p">)</span></div>
+
+ <span class="k">def</span> <span class="nf">_retrieve_mails_attachments_by_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">mail_folder</span><span class="p">,</span> <span class="n">check_regex</span><span class="p">,</span>
+ <span class="n">latest_only</span><span class="p">):</span>
+ <span class="n">all_matching_attachments</span> <span class="o">=</span> <span class="p">[]</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">mail_client</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">mail_folder</span><span class="p">)</span>
+
+ <span class="k">for</span> <span class="n">mail_id</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_list_mail_ids_desc</span><span class="p">():</span>
+ <span class="n">response_mail_body</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_fetch_mail_body</span><span class="p">(</span><span class="n">mail_id</span><span class="p">)</span>
+ <span class="n">matching_attachments</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_check_mail_body</span><span class="p">(</span><span class="n">response_mail_body</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">check_regex</span><span class="p">,</span> <span class="n">latest_only</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">matching_attachments</span><span class="p">:</span>
+ <span class="n">all_matching_attachments</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">matching_attachments</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">latest_only</span><span class="p">:</span>
+ <span class="k">break</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">mail_client</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
+
+ <span class="k">return</span> <span class="n">all_matching_attachments</span>
+
+ <span class="k">def</span> <span class="nf">_list_mail_ids_desc</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="n">result</span><span class="p">,</span> <span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">mail_client</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="s1">'All'</span><span class="p">)</span>
+ <span class="n">mail_ids</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">()</span>
+ <span class="k">return</span> <span class="nb">reversed</span><span class="p">(</span><span class="n">mail_ids</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_fetch_mail_body</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">mail_id</span><span class="p">):</span>
+ <span class="n">result</span><span class="p">,</span> <span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">mail_client</span><span class="o">.</span><span class="n">fetch</span><span class="p">(</span><span class="n">mail_id</span><span class="p">,</span> <span class="s1">'(RFC822)'</span><span class="p">)</span>
+ <span class="n">mail_body</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span> <span class="c1"># The mail body is always in this specific location</span>
+ <span class="n">mail_body_str</span> <span class="o">=</span> <span class="n">mail_body</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">mail_body_str</span>
+
+ <span class="k">def</span> <span class="nf">_check_mail_body</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">response_mail_body</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">check_regex</span><span class="p">,</span> <span class="n">latest_only</span><span class="p">):</span>
+ <span class="n">mail</span> <span class="o">=</span> <span class="n">Mail</span><span class="p">(</span><span class="n">response_mail_body</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">mail</span><span class="o">.</span><span class="n">has_attachments</span><span class="p">():</span>
+ <span class="k">return</span> <span class="n">mail</span><span class="o">.</span><span class="n">get_attachments_by_name</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">check_regex</span><span class="p">,</span> <span class="n">find_first</span><span class="o">=</span><span class="n">latest_only</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_create_files</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">mail_attachments</span><span class="p">,</span> <span class="n">local_output_directory</span><span class="p">):</span>
+ <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">payload</span> <span class="ow">in</span> <span class="n">mail_attachments</span><span class="p">:</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_is_symlink</span><span class="p">(</span><span class="n">name</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">'Can not create file because it is a symlink!'</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_is_escaping_current_directory</span><span class="p">(</span><span class="n">name</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">'Can not create file because it is escaping the current directory!'</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_create_file</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">payload</span><span class="p">,</span> <span class="n">local_output_directory</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_is_symlink</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">islink</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_is_escaping_current_directory</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
+ <span class="k">return</span> <span class="s1">'../'</span> <span class="ow">in</span> <span class="n">name</span>
+
+ <span class="k">def</span> <span class="nf">_correct_path</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">local_output_directory</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">local_output_directory</span> <span class="o">+</span> <span class="n">name</span> <span class="k">if</span> <span class="n">local_output_directory</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s1">'/'</span><span class="p">)</span> \
+ <span class="k">else</span> <span class="n">local_output_directory</span> <span class="o">+</span> <span class="s1">'/'</span> <span class="o">+</span> <span class="n">name</span>
+
+ <span class="k">def</span> <span class="nf">_create_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">payload</span><span class="p">,</span> <span class="n">local_output_directory</span><span class="p">):</span>
+ <span class="n">file_path</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_correct_path</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">local_output_directory</span><span class="p">)</span>
+
+ <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">file_path</span><span class="p">,</span> <span class="s1">'wb'</span><span class="p">)</span> <span class="k">as</span> <span class="n">file</span><span class="p">:</span>
+ <span class="n">file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span></div>
+
+
+<span class="k">class</span> <span class="nc">Mail</span><span class="p">(</span><span class="n">LoggingMixin</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> This class simplifies working with mails returned by the imaplib client.</span>
+
+<span class="sd"> :param mail_body: The mail body of a mail received from imaplib client.</span>
+<span class="sd"> :type mail_body: str</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">mail_body</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">Mail</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mail</span> <span class="o">=</span> <span class="n">email</span><span class="o">.</span><span class="n">message_from_string</span><span class="p">(</span><span class="n">mail_body</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">has_attachments</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Checks the mail for a attachments.</span>
+
+<span class="sd"> :returns: True if it has attachments and False if not.</span>
+<span class="sd"> :rtype: bool</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">mail</span><span class="o">.</span><span class="n">get_content_maintype</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'multipart'</span>
+
+ <span class="k">def</span> <span class="nf">get_attachments_by_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">check_regex</span><span class="p">,</span> <span class="n">find_first</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Gets all attachments by name for the mail.</span>
+
+<span class="sd"> :param name: The name of the attachment to look for.</span>
+<span class="sd"> :type name: str</span>
+<span class="sd"> :param check_regex: Checks the name for a regular expression.</span>
+<span class="sd"> :type check_regex: bool</span>
+<span class="sd"> :param find_first: If set to True it will only find the first match and then quit.</span>
+<span class="sd"> The default value is False.</span>
+<span class="sd"> :type find_first: bool</span>
+<span class="sd"> :returns: a list of tuples each containing name and payload</span>
+<span class="sd"> where the attachments name matches the given name.</span>
+<span class="sd"> :rtype: list of tuple</span>
+<span class="sd"> """</span>
+ <span class="n">attachments</span> <span class="o">=</span> <span class="p">[]</span>
+
+ <span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">mail</span><span class="o">.</span><span class="n">walk</span><span class="p">():</span>
+ <span class="n">mail_part</span> <span class="o">=</span> <span class="n">MailPart</span><span class="p">(</span><span class="n">part</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">mail_part</span><span class="o">.</span><span class="n">is_attachment</span><span class="p">():</span>
+ <span class="n">found_attachment</span> <span class="o">=</span> <span class="n">mail_part</span><span class="o">.</span><span class="n">has_matching_name</span><span class="p">(</span><span class="n">name</span><span class="p">)</span> <span class="k">if</span> <span class="n">check_regex</span> \
+ <span class="k">else</span> <span class="n">mail_part</span><span class="o">.</span><span class="n">has_equal_name</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">found_attachment</span><span class="p">:</span>
+ <span class="n">file_name</span><span class="p">,</span> <span class="n">file_payload</span> <span class="o">=</span> <span class="n">mail_part</span><span class="o">.</span><span class="n">get_file</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Found attachment: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">file_name</span><span class="p">))</span>
+ <span class="n">attachments</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">file_name</span><span class="p">,</span> <span class="n">file_payload</span><span class="p">))</span>
+ <span class="k">if</span> <span class="n">find_first</span><span class="p">:</span>
+ <span class="k">break</span>
+
+ <span class="k">return</span> <span class="n">attachments</span>
+
+
+<span class="k">class</span> <span class="nc">MailPart</span><span class="p">:</span>
+ <span class="sd">"""</span>
+<span class="sd"> This class is a wrapper for a Mail object's part and gives it more features.</span>
+
+<span class="sd"> :param part: The mail part in a Mail object.</span>
+<span class="sd"> :type part: any</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">part</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">part</span> <span class="o">=</span> <span class="n">part</span>
+
+ <span class="k">def</span> <span class="nf">is_attachment</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Checks if the part is a valid mail attachment.</span>
+
+<span class="sd"> :returns: True if it is an attachment and False if not.</span>
+<span class="sd"> :rtype: bool</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">part</span><span class="o">.</span><span class="n">get_content_maintype</span><span class="p">()</span> <span class="o">!=</span> <span class="s1">'multipart'</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">part</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Content-D [...]
+
+ <span class="k">def</span> <span class="nf">has_matching_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Checks if the given name matches the part's name.</span>
+
+<span class="sd"> :param name: The name to look for.</span>
+<span class="sd"> :type name: str</span>
+<span class="sd"> :returns: True if it matches the name (including regular expression).</span>
+<span class="sd"> :rtype: tuple</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">part</span><span class="o">.</span><span class="n">get_filename</span><span class="p">())</span>
+
+ <span class="k">def</span> <span class="nf">has_equal_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Checks if the given name is equal to the part's name.</span>
+
+<span class="sd"> :param name: The name to look for.</span>
+<span class="sd"> :type name: str</span>
+<span class="sd"> :returns: True if it is equal to the given name.</span>
+<span class="sd"> :rtype: bool</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">part</span><span class="o">.</span><span class="n">get_filename</span><span class="p">()</span> <span class="o">==</span> <span class="n">name</span>
+
+ <span class="k">def</span> <span class="nf">get_file</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Gets the file including name and payload.</span>
+
+<span class="sd"> :returns: the part's name and payload.</span>
+<span class="sd"> :rtype: tuple</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">part</span><span class="o">.</span><span class="n">get_filename</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">part</span><span class="o">.</span><span class="n">get_payload</span><span class="p">(</span><span class="n">decode</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/jenkins_hook.html b/_modules/airflow/contrib/hooks/jenkins_hook.html
index 1ffe07f..ccbadeb 100644
--- a/_modules/airflow/contrib/hooks/jenkins_hook.html
+++ b/_modules/airflow/contrib/hooks/jenkins_hook.html
@@ -237,20 +237,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/jira_hook.html b/_modules/airflow/contrib/hooks/jira_hook.html
index cdf8dc2..308a711 100644
--- a/_modules/airflow/contrib/hooks/jira_hook.html
+++ b/_modules/airflow/contrib/hooks/jira_hook.html
@@ -276,20 +276,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/mongo_hook.html b/_modules/airflow/contrib/hooks/mongo_hook.html
index e1c22e1..536ac27 100644
--- a/_modules/airflow/contrib/hooks/mongo_hook.html
+++ b/_modules/airflow/contrib/hooks/mongo_hook.html
@@ -308,20 +308,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/pinot_hook.html b/_modules/airflow/contrib/hooks/pinot_hook.html
index 2c03016..8e27b62 100644
--- a/_modules/airflow/contrib/hooks/pinot_hook.html
+++ b/_modules/airflow/contrib/hooks/pinot_hook.html
@@ -294,20 +294,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/qubole_hook.html b/_modules/airflow/contrib/hooks/qubole_hook.html
index 4968269..0f96d97 100644
--- a/_modules/airflow/contrib/hooks/qubole_hook.html
+++ b/_modules/airflow/contrib/hooks/qubole_hook.html
@@ -403,20 +403,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/redis_hook.html b/_modules/airflow/contrib/hooks/redis_hook.html
index e559ff0..760cf29 100644
--- a/_modules/airflow/contrib/hooks/redis_hook.html
+++ b/_modules/airflow/contrib/hooks/redis_hook.html
@@ -282,20 +282,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/redshift_hook.html b/_modules/airflow/contrib/hooks/redshift_hook.html
index e587dc2..d2c6a53 100644
--- a/_modules/airflow/contrib/hooks/redshift_hook.html
+++ b/_modules/airflow/contrib/hooks/redshift_hook.html
@@ -302,20 +302,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/sagemaker_hook.html b/_modules/airflow/contrib/hooks/sagemaker_hook.html
new file mode 100644
index 0000000..d501097
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/sagemaker_hook.html
@@ -0,0 +1,972 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.sagemaker_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.sagemaker_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.sagemaker_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">import</span> <span class="nn">tarfile</span>
+<span class="kn">import</span> <span class="nn">tempfile</span>
+<span class="kn">import</span> <span class="nn">time</span>
+<span class="kn">import</span> <span class="nn">os</span>
+<span class="kn">import</span> <span class="nn">collections</span>
+
+<span class="kn">import</span> <span class="nn">botocore.config</span>
+<span class="kn">from</span> <span class="nn">botocore.exceptions</span> <span class="k">import</span> <span class="n">ClientError</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.aws_hook</span> <span class="k">import</span> <span class="n">AwsHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.S3_hook</span> <span class="k">import</span> <span class="n">S3Hook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils</span> <span class="k">import</span> <span class="n">timezone</span>
+
+
+<span class="k">class</span> <span class="nc">LogState</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+ <span class="n">STARTING</span> <span class="o">=</span> <span class="mi">1</span>
+ <span class="n">WAIT_IN_PROGRESS</span> <span class="o">=</span> <span class="mi">2</span>
+ <span class="n">TAILING</span> <span class="o">=</span> <span class="mi">3</span>
+ <span class="n">JOB_COMPLETE</span> <span class="o">=</span> <span class="mi">4</span>
+ <span class="n">COMPLETE</span> <span class="o">=</span> <span class="mi">5</span>
+
+
+<span class="c1"># Position is a tuple that includes the last read timestamp and the number of items that were read</span>
+<span class="c1"># at that time. This is used to figure out which event to start with on the next read.</span>
+<span class="n">Position</span> <span class="o">=</span> <span class="n">collections</span><span class="o">.</span><span class="n">namedtuple</span><span class="p">(</span><span class="s1">'Position'</span><span class="p">,</span> <span class="p">[</span><span class="s1">'timestamp'</span><span class="p">,</span> <span class="s1">'skip'</span><span class="p">])</span>
+
+
+<span class="k">def</span> <span class="nf">argmin</span><span class="p">(</span><span class="n">arr</span><span class="p">,</span> <span class="n">f</span><span class="p">):</span>
+ <span class="sd">"""Return the index, i, in arr that minimizes f(arr[i])"""</span>
+ <span class="n">m</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="n">i</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">item</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">arr</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">item</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">m</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">f</span><span class="p">(</span><span class="n">item</span><span class="p">)</span> <span class="o"><</span> <span class="n">m</span><span class="p">:</span>
+ <span class="n">m</span> <span class="o">=</span> <span class="n">f</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
+ <span class="n">i</span> <span class="o">=</span> <span class="n">idx</span>
+ <span class="k">return</span> <span class="n">i</span>
+
+
+<span class="k">def</span> <span class="nf">secondary_training_status_changed</span><span class="p">(</span><span class="n">current_job_description</span><span class="p">,</span> <span class="n">prev_job_description</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns true if training job's secondary status message has changed.</span>
+
+<span class="sd"> :param current_job_description: Current job description, returned from DescribeTrainingJob call.</span>
+<span class="sd"> :type current_job_description: dict</span>
+<span class="sd"> :param prev_job_description: Previous job description, returned from DescribeTrainingJob call.</span>
+<span class="sd"> :type prev_job_description: dict</span>
+
+<span class="sd"> :return: Whether the secondary status message of a training job changed or not.</span>
+<span class="sd"> """</span>
+ <span class="n">current_secondary_status_transitions</span> <span class="o">=</span> <span class="n">current_job_description</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'SecondaryStatusTransitions'</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">current_secondary_status_transitions</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="nb">len</span><span class="p">(</span><span class="n">current_secondary_status_transitions</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">False</span>
+
+ <span class="n">prev_job_secondary_status_transitions</span> <span class="o">=</span> <span class="n">prev_job_description</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'SecondaryStatusTransitions'</span><span class="p">)</span> \
+ <span class="k">if</span> <span class="n">prev_job_description</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="kc">None</span>
+
+ <span class="n">last_message</span> <span class="o">=</span> <span class="n">prev_job_secondary_status_transitions</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">][</span><span class="s1">'StatusMessage'</span><span class="p">]</span> \
+ <span class="k">if</span> <span class="n">prev_job_secondary_status_transitions</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> \
+ <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">prev_job_secondary_status_transitions</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span> <span class="k">else</span> <span class="s1">''</span>
+
+ <span class="n">message</span> <span class="o">=</span> <span class="n">current_job_description</span><span class="p">[</span><span class="s1">'SecondaryStatusTransitions'</span><span class="p">][</span><span class="o">-</span><span class="mi">1</span><span class="p">][</span><span class="s1">'StatusMessage'</span><span class="p">]</span>
+
+ <span class="k">return</span> <span class="n">message</span> <span class="o">!=</span> <span class="n">last_message</span>
+
+
+<span class="k">def</span> <span class="nf">secondary_training_status_message</span><span class="p">(</span><span class="n">job_description</span><span class="p">,</span> <span class="n">prev_description</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns a string contains start time and the secondary training job status message.</span>
+
+<span class="sd"> :param job_description: Returned response from DescribeTrainingJob call</span>
+<span class="sd"> :type job_description: dict</span>
+<span class="sd"> :param prev_description: Previous job description from DescribeTrainingJob call</span>
+<span class="sd"> :type prev_description: dict</span>
+
+<span class="sd"> :return: Job status string to be printed.</span>
+<span class="sd"> """</span>
+
+ <span class="k">if</span> <span class="n">job_description</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">job_description</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'SecondaryStatusTransitions'</span><span class="p">)</span> <span class="ow">is</span> <span class="kc">None</span>\
+ <span class="ow">or</span> <span class="nb">len</span><span class="p">(</span><span class="n">job_description</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'SecondaryStatusTransitions'</span><span class="p">))</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="k">return</span> <span class="s1">''</span>
+
+ <span class="n">prev_description_secondary_transitions</span> <span class="o">=</span> <span class="n">prev_description</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'SecondaryStatusTransitions'</span><span class="p">)</span>\
+ <span class="k">if</span> <span class="n">prev_description</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="kc">None</span>
+ <span class="n">prev_transitions_num</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">prev_description</span><span class="p">[</span><span class="s1">'SecondaryStatusTransitions'</span><span class="p">])</span>\
+ <span class="k">if</span> <span class="n">prev_description_secondary_transitions</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
+ <span class="n">current_transitions</span> <span class="o">=</span> <span class="n">job_description</span><span class="p">[</span><span class="s1">'SecondaryStatusTransitions'</span><span class="p">]</span>
+
+ <span class="n">transitions_to_print</span> <span class="o">=</span> <span class="n">current_transitions</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">:]</span> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">current_transitions</span><span class="p">)</span> <span class="o">==</span> <span class="n">prev_transitions_num</span> <span class="k">else</span> \
+ <span class="n">current_transitions</span><span class="p">[</span><span class="n">prev_transitions_num</span> <span class="o">-</span> <span class="nb">len</span><span class="p">(</span><span class="n">current_transitions</span><span class="p">):]</span>
+
+ <span class="n">status_strs</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="k">for</span> <span class="n">transition</span> <span class="ow">in</span> <span class="n">transitions_to_print</span><span class="p">:</span>
+ <span class="n">message</span> <span class="o">=</span> <span class="n">transition</span><span class="p">[</span><span class="s1">'StatusMessage'</span><span class="p">]</span>
+ <span class="n">time_str</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">convert_to_utc</span><span class="p">(</span><span class="n">job_description</span><span class="p">[</span><span class="s1">'LastModifiedTime'</span><span class="p">])</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s1">'%Y-%m-</span><span class="si">%d</span><span class="s1"> %H:%M:%S'</s [...]
+ <span class="n">status_strs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s1">'</span><span class="si">{}</span><span class="s1"> </span><span class="si">{}</span><span class="s1"> - </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">time_str</span><span class="p">,</span> <span class="n">transition</span><span class="p">[</ [...]
+
+ <span class="k">return</span> <span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">status_strs</span><span class="p">)</span>
+
+
+<div class="viewcode-block" id="SageMakerHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook">[docs]</a><span class="k">class</span> <span class="nc">SageMakerHook</span><span class="p">(</span><span class="n">AwsHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with Amazon SageMaker.</span>
+<span class="sd"> """</span>
+ <span class="n">non_terminal_states</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'InProgress'</span><span class="p">,</span> <span class="s1">'Stopping'</span><span class="p">}</span>
+ <span class="n">endpoint_non_terminal_states</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'Creating'</span><span class="p">,</span> <span class="s1">'Updating'</span><span class="p">,</span> <span class="s1">'SystemUpdating'</span><span class="p">,</span>
+ <span class="s1">'RollingBack'</span><span class="p">,</span> <span class="s1">'Deleting'</span><span class="p">}</span>
+ <span class="n">failed_states</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'Failed'</span><span class="p">}</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">SageMakerHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">s3_hook</span> <span class="o">=</span> <span class="n">S3Hook</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span>
+
+<div class="viewcode-block" id="SageMakerHook.tar_and_s3_upload"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.tar_and_s3_upload">[docs]</a> <span class="k">def</span> <span class="nf">tar_and_s3_upload</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">bucket</span><span class=" [...]
+ <span class="sd">"""</span>
+<span class="sd"> Tar the local file or directory and upload to s3</span>
+
+<span class="sd"> :param path: local file or directory</span>
+<span class="sd"> :type path: str</span>
+<span class="sd"> :param key: s3 key</span>
+<span class="sd"> :type key: str</span>
+<span class="sd"> :param bucket: s3 bucket</span>
+<span class="sd"> :type bucket: str</span>
+<span class="sd"> :return: None</span>
+<span class="sd"> """</span>
+ <span class="k">with</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">TemporaryFile</span><span class="p">()</span> <span class="k">as</span> <span class="n">temp_file</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isdir</span><span class="p">(</span><span class="n">path</span><span class="p">):</span>
+ <span class="n">files</span> <span class="o">=</span> <span class="p">[</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">name</span><span class="p">)</span> <span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class= [...]
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">files</span> <span class="o">=</span> <span class="p">[</span><span class="n">path</span><span class="p">]</span>
+ <span class="k">with</span> <span class="n">tarfile</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">mode</span><span class="o">=</span><span class="s1">'w:gz'</span><span class="p">,</span> <span class="n">fileobj</span><span class="o">=</span><span class="n">temp_file</span><span class="p">)</span> <span class="k">as</span> <span class="n">tar_file</span><span class="p">:</span>
+ <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">files</span><span class="p">:</span>
+ <span class="n">tar_file</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">f</span><span class="p">,</span> <span class="n">arcname</span><span class="o">=</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">basename</span><span class="p">(</span><span class="n">f</span><span class="p">))</span>
+ <span class="n">temp_file</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">s3_hook</span><span class="o">.</span><span class="n">load_file_obj</span><span class="p">(</span><span class="n">temp_file</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="n">replace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="SageMakerHook.configure_s3_resources"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.configure_s3_resources">[docs]</a> <span class="k">def</span> <span class="nf">configure_s3_resources</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">config</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Extract the S3 operations from the configuration and execute them.</span>
+
+<span class="sd"> :param config: config of SageMaker operation</span>
+<span class="sd"> :type config: dict</span>
+<span class="sd"> :return: dict</span>
+<span class="sd"> """</span>
+ <span class="n">s3_operations</span> <span class="o">=</span> <span class="n">config</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s1">'S3Operations'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">s3_operations</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">create_bucket_ops</span> <span class="o">=</span> <span class="n">s3_operations</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'S3CreateBucket'</span><span class="p">,</span> <span class="p">[])</span>
+ <span class="n">upload_ops</span> <span class="o">=</span> <span class="n">s3_operations</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'S3Upload'</span><span class="p">,</span> <span class="p">[])</span>
+ <span class="k">for</span> <span class="n">op</span> <span class="ow">in</span> <span class="n">create_bucket_ops</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">s3_hook</span><span class="o">.</span><span class="n">create_bucket</span><span class="p">(</span><span class="n">bucket_name</span><span class="o">=</span><span class="n">op</span><span class="p">[</span><span class="s1">'Bucket'</span><span class="p">])</span>
+ <span class="k">for</span> <span class="n">op</span> <span class="ow">in</span> <span class="n">upload_ops</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">op</span><span class="p">[</span><span class="s1">'Tar'</span><span class="p">]:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">tar_and_s3_upload</span><span class="p">(</span><span class="n">op</span><span class="p">[</span><span class="s1">'Path'</span><span class="p">],</span> <span class="n">op</span><span class="p">[</span><span class="s1">'Key'</span><span class="p">],</span>
+ <span class="n">op</span><span class="p">[</span><span class="s1">'Bucket'</span><span class="p">])</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">s3_hook</span><span class="o">.</span><span class="n">load_file</span><span class="p">(</span><span class="n">op</span><span class="p">[</span><span class="s1">'Path'</span><span class="p">],</span> <span class="n">op</span><span class="p">[</span><span class="s1">'Key'</span><span class="p">],</span>
+ <span class="n">op</span><span class="p">[</span><span class="s1">'Bucket'</span><span class="p">])</span></div>
+
+<div class="viewcode-block" id="SageMakerHook.check_s3_url"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.check_s3_url">[docs]</a> <span class="k">def</span> <span class="nf">check_s3_url</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">s3url</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Check if an S3 URL exists</span>
+
+<span class="sd"> :param s3url: S3 url</span>
+<span class="sd"> :type s3url:str</span>
+<span class="sd"> :return: bool</span>
+<span class="sd"> """</span>
+ <span class="n">bucket</span><span class="p">,</span> <span class="n">key</span> <span class="o">=</span> <span class="n">S3Hook</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="n">s3url</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">s3_hook</span><span class="o">.</span><span class="n">check_for_bucket</span><span class="p">(</span><span class="n">bucket_name</span><span class="o">=</span><span class="n">bucket</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s2">"The input S3 Bucket </span><span class="si">{}</span><span class="s2"> does not exist "</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">bucket</span><span class="p">))</span>
+ <span class="k">if</span> <span class="n">key</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">s3_hook</span><span class="o">.</span><span class="n">check_for_key</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="n">bucket</span><span class="p">)</span>\
+ <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">s3_hook</span><span class="o">.</span><span class="n">check_for_prefix</span><span class="p">(</span>
+ <span class="n">prefix</span><span class="o">=</span><span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="n">bucket</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">'/'</span><span class="p">):</span>
+ <span class="c1"># check if s3 key exists in the case user provides a single file</span>
+ <span class="c1"># or if s3 prefix exists in the case user provides multiple files in</span>
+ <span class="c1"># a prefix</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"The input S3 Key "</span>
+ <span class="s2">"or Prefix </span><span class="si">{}</span><span class="s2"> does not exist in the Bucket </span><span class="si">{}</span><span class="s2">"</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">s3url</span><span class="p">,</span> <span class="n">bucket</span><span class="p">))</span>
+ <span class="k">return</span> <span class="kc">True</span></div>
+
+<div class="viewcode-block" id="SageMakerHook.check_training_config"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.check_training_config">[docs]</a> <span class="k">def</span> <span class="nf">check_training_config</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">training_config</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Check if a training configuration is valid</span>
+
+<span class="sd"> :param training_config: training_config</span>
+<span class="sd"> :type training_config: dict</span>
+<span class="sd"> :return: None</span>
+<span class="sd"> """</span>
+ <span class="k">for</span> <span class="n">channel</span> <span class="ow">in</span> <span class="n">training_config</span><span class="p">[</span><span class="s1">'InputDataConfig'</span><span class="p">]:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">check_s3_url</span><span class="p">(</span><span class="n">channel</span><span class="p">[</span><span class="s1">'DataSource'</span><span class="p">][</span><span class="s1">'S3DataSource'</span><span class="p">][</span><span class="s1">'S3Uri'</span><span class="p">])</span></div>
+
+<div class="viewcode-block" id="SageMakerHook.check_tuning_config"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.check_tuning_config">[docs]</a> <span class="k">def</span> <span class="nf">check_tuning_config</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tuning_config</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Check if a tuning configuration is valid</span>
+
+<span class="sd"> :param tuning_config: tuning_config</span>
+<span class="sd"> :type tuning_config: dict</span>
+<span class="sd"> :return: None</span>
+<span class="sd"> """</span>
+ <span class="k">for</span> <span class="n">channel</span> <span class="ow">in</span> <span class="n">tuning_config</span><span class="p">[</span><span class="s1">'TrainingJobDefinition'</span><span class="p">][</span><span class="s1">'InputDataConfig'</span><span class="p">]:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">check_s3_url</span><span class="p">(</span><span class="n">channel</span><span class="p">[</span><span class="s1">'DataSource'</span><span class="p">][</span><span class="s1">'S3DataSource'</span><span class="p">][</span><span class="s1">'S3Uri'</span><span class="p">])</span></div>
+
+<div class="viewcode-block" id="SageMakerHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Establish an AWS connection for SageMaker</span>
+
+<span class="sd"> :return: a boto3 SageMaker client</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_client_type</span><span class="p">(</span><span class="s1">'sagemaker'</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="SageMakerHook.get_log_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.get_log_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_log_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Establish an AWS connection for retrieving logs during training</span>
+
+<span class="sd"> :return: a boto3 CloudWatchLog client</span>
+<span class="sd"> """</span>
+ <span class="n">config</span> <span class="o">=</span> <span class="n">botocore</span><span class="o">.</span><span class="n">config</span><span class="o">.</span><span class="n">Config</span><span class="p">(</span><span class="n">retries</span><span class="o">=</span><span class="p">{</span><span class="s1">'max_attempts'</span><span class="p">:</span> <span class="mi">15</span><span class="p">})</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_client_type</span><span class="p">(</span><span class="s1">'logs'</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="n">config</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="SageMakerHook.log_stream"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.log_stream">[docs]</a> <span class="k">def</span> <span class="nf">log_stream</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">log_group</span><span class="p">,</span> <span class="n">stream_name</span><span class="p">,</span> <span class="n">start_time</span><span class="o">= [...]
+ <span class="sd">"""</span>
+<span class="sd"> A generator for log items in a single stream. This will yield all the</span>
+<span class="sd"> items that are available at the current moment.</span>
+
+<span class="sd"> :param log_group: The name of the log group.</span>
+<span class="sd"> :type log_group: str</span>
+<span class="sd"> :param stream_name: The name of the specific stream.</span>
+<span class="sd"> :type stream_name: str</span>
+<span class="sd"> :param start_time: The time stamp value to start reading the logs from (default: 0).</span>
+<span class="sd"> :type start_time: int</span>
+<span class="sd"> :param skip: The number of log entries to skip at the start (default: 0).</span>
+<span class="sd"> This is for when there are multiple entries at the same timestamp.</span>
+<span class="sd"> :type skip: int</span>
+<span class="sd"> :return:A CloudWatch log event with the following key-value pairs:</span>
+<span class="sd"> 'timestamp' (int): The time in milliseconds of the event.</span>
+<span class="sd"> 'message' (str): The log event data.</span>
+<span class="sd"> 'ingestionTime' (int): The time in milliseconds the event was ingested.</span>
+<span class="sd"> """</span>
+
+ <span class="n">next_token</span> <span class="o">=</span> <span class="kc">None</span>
+
+ <span class="n">event_count</span> <span class="o">=</span> <span class="mi">1</span>
+ <span class="k">while</span> <span class="n">event_count</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">next_token</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">token_arg</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'nextToken'</span><span class="p">:</span> <span class="n">next_token</span><span class="p">}</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">token_arg</span> <span class="o">=</span> <span class="p">{}</span>
+
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_log_conn</span><span class="p">()</span><span class="o">.</span><span class="n">get_log_events</span><span class="p">(</span><span class="n">logGroupName</span><span class="o">=</span><span class="n">log_group</span><span class="p">,</span>
+ <span class="n">logStreamName</span><span class="o">=</span><span class="n">stream_name</span><span class="p">,</span>
+ <span class="n">startTime</span><span class="o">=</span><span class="n">start_time</span><span class="p">,</span>
+ <span class="n">startFromHead</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">token_arg</span><span class="p">)</span>
+ <span class="n">next_token</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">'nextForwardToken'</span><span class="p">]</span>
+ <span class="n">events</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">'events'</span><span class="p">]</span>
+ <span class="n">event_count</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">events</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">event_count</span> <span class="o">></span> <span class="n">skip</span><span class="p">:</span>
+ <span class="n">events</span> <span class="o">=</span> <span class="n">events</span><span class="p">[</span><span class="n">skip</span><span class="p">:]</span>
+ <span class="n">skip</span> <span class="o">=</span> <span class="mi">0</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">skip</span> <span class="o">=</span> <span class="n">skip</span> <span class="o">-</span> <span class="n">event_count</span>
+ <span class="n">events</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="k">for</span> <span class="n">ev</span> <span class="ow">in</span> <span class="n">events</span><span class="p">:</span>
+ <span class="k">yield</span> <span class="n">ev</span></div>
+
+<div class="viewcode-block" id="SageMakerHook.multi_stream_iter"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.multi_stream_iter">[docs]</a> <span class="k">def</span> <span class="nf">multi_stream_iter</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">log_group</span><span class="p">,</span> <span class="n">streams</span><span class="p">,</span> <span class="n">positions</span>< [...]
+ <span class="sd">"""</span>
+<span class="sd"> Iterate over the available events coming from a set of log streams in a single log group</span>
+<span class="sd"> interleaving the events from each stream so they're yielded in timestamp order.</span>
+
+<span class="sd"> :param log_group: The name of the log group.</span>
+<span class="sd"> :type log_group: str</span>
+<span class="sd"> :param streams: A list of the log stream names. The position of the stream in this list is</span>
+<span class="sd"> the stream number.</span>
+<span class="sd"> :type streams: list</span>
+<span class="sd"> :param positions: A list of pairs of (timestamp, skip) which represents the last record</span>
+<span class="sd"> read from each stream.</span>
+<span class="sd"> :type positions: list</span>
+<span class="sd"> :return: A tuple of (stream number, cloudwatch log event).</span>
+<span class="sd"> """</span>
+ <span class="n">positions</span> <span class="o">=</span> <span class="n">positions</span> <span class="ow">or</span> <span class="p">{</span><span class="n">s</span><span class="p">:</span> <span class="n">Position</span><span class="p">(</span><span class="n">timestamp</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">skip</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span> <span class="k">for</span> <spa [...]
+ <span class="n">event_iters</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">log_stream</span><span class="p">(</span><span class="n">log_group</span><span class="p">,</span> <span class="n">s</span><span class="p">,</span> <span class="n">positions</span><span class="p">[</span><span class="n">s</span><span class="p">]</span><span class="o">.</span><span class="n">timestamp</span><span class="p">,</span> [...]
+ <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">streams</span><span class="p">]</span>
+ <span class="n">events</span> <span class="o">=</span> <span class="p">[</span><span class="nb">next</span><span class="p">(</span><span class="n">s</span><span class="p">)</span> <span class="k">if</span> <span class="n">s</span> <span class="k">else</span> <span class="kc">None</span> <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">event_iters</span><span class="p">]</span>
+
+ <span class="k">while</span> <span class="nb">any</span><span class="p">(</span><span class="n">events</span><span class="p">):</span>
+ <span class="n">i</span> <span class="o">=</span> <span class="n">argmin</span><span class="p">(</span><span class="n">events</span><span class="p">,</span> <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="s1">'timestamp'</span><span class="p">]</span> <span class="k">if</span> <span class="n">x</span> <span class="k">else</span> <span class="mi">9999999999</span><span class="p" [...]
+ <span class="k">yield</span> <span class="p">(</span><span class="n">i</span><span class="p">,</span> <span class="n">events</span><span class="p">[</span><span class="n">i</span><span class="p">])</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">events</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span><span class="n">event_iters</span><span class="p">[</span><span class="n">i</span><span class="p">])</span>
+ <span class="k">except</span> <span class="ne">StopIteration</span><span class="p">:</span>
+ <span class="n">events</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span></div>
+
+<div class="viewcode-block" id="SageMakerHook.create_training_job"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.create_training_job">[docs]</a> <span class="k">def</span> <span class="nf">create_training_job</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">config</span><span class="p">,</span> <span class="n">wait_for_completion</span><span class="o">=</span><span class="kc">Tr [...]
+ <span class="n">check_interval</span><span class="o">=</span><span class="mi">30</span><span class="p">,</span> <span class="n">max_ingestion_time</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Create a training job</span>
+
+<span class="sd"> :param config: the config for training</span>
+<span class="sd"> :type config: dict</span>
+<span class="sd"> :param wait_for_completion: if the program should keep running until job finishes</span>
+<span class="sd"> :type wait_for_completion: bool</span>
+<span class="sd"> :param check_interval: the time interval in seconds which the operator</span>
+<span class="sd"> will check the status of any SageMaker job</span>
+<span class="sd"> :type check_interval: int</span>
+<span class="sd"> :param max_ingestion_time: the maximum ingestion time in seconds. Any</span>
+<span class="sd"> SageMaker jobs that run longer than this will fail. Setting this to</span>
+<span class="sd"> None implies no timeout for any SageMaker job.</span>
+<span class="sd"> :type max_ingestion_time: int</span>
+<span class="sd"> :return: A response to training job creation</span>
+<span class="sd"> """</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">check_training_config</span><span class="p">(</span><span class="n">config</span><span class="p">)</span>
+
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">create_training_job</span><span class="p">(</span><span class="o">**</span><span class="n">config</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">print_log</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">check_training_status_with_log</span><span class="p">(</span><span class="n">config</span><span class="p">[</span><span class="s1">'TrainingJobName'</span><span class="p">],</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">non_terminal_states</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">failed_states</span><span class="p">,</span>
+ <span class="n">wait_for_completion</span><span class="p">,</span>
+ <span class="n">check_interval</span><span class="p">,</span> <span class="n">max_ingestion_time</span>
+ <span class="p">)</span>
+ <span class="k">elif</span> <span class="n">wait_for_completion</span><span class="p">:</span>
+ <span class="n">describe_response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">check_status</span><span class="p">(</span><span class="n">config</span><span class="p">[</span><span class="s1">'TrainingJobName'</span><span class="p">],</span>
+ <span class="s1">'TrainingJobStatus'</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">describe_training_job</span><span class="p">,</span>
+ <span class="n">check_interval</span><span class="p">,</span> <span class="n">max_ingestion_time</span>
+ <span class="p">)</span>
+
+ <span class="n">billable_time</span> <span class="o">=</span> \
+ <span class="p">(</span><span class="n">describe_response</span><span class="p">[</span><span class="s1">'TrainingEndTime'</span><span class="p">]</span> <span class="o">-</span> <span class="n">describe_response</span><span class="p">[</span><span class="s1">'TrainingStartTime'</span><span class="p">])</span> <span class="o">*</span> \
+ <span class="n">describe_response</span><span class="p">[</span><span class="s1">'ResourceConfig'</span><span class="p">][</span><span class="s1">'InstanceCount'</span><span class="p">]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Billable seconds:</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">billable_time</span><span class="o">.</span><span class="n">total_seconds</span><s [...]
+
+ <span class="k">return</span> <span class="n">response</span></div>
+
+<div class="viewcode-block" id="SageMakerHook.create_tuning_job"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.create_tuning_job">[docs]</a> <span class="k">def</span> <span class="nf">create_tuning_job</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">config</span><span class="p">,</span> <span class="n">wait_for_completion</span><span class="o">=</span><span class="kc">True</sp [...]
+ <span class="n">check_interval</span><span class="o">=</span><span class="mi">30</span><span class="p">,</span> <span class="n">max_ingestion_time</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Create a tuning job</span>
+
+<span class="sd"> :param config: the config for tuning</span>
+<span class="sd"> :type config: dict</span>
+<span class="sd"> :param wait_for_completion: if the program should keep running until job finishes</span>
+<span class="sd"> :param wait_for_completion: bool</span>
+<span class="sd"> :param check_interval: the time interval in seconds which the operator</span>
+<span class="sd"> will check the status of any SageMaker job</span>
+<span class="sd"> :type check_interval: int</span>
+<span class="sd"> :param max_ingestion_time: the maximum ingestion time in seconds. Any</span>
+<span class="sd"> SageMaker jobs that run longer than this will fail. Setting this to</span>
+<span class="sd"> None implies no timeout for any SageMaker job.</span>
+<span class="sd"> :type max_ingestion_time: int</span>
+<span class="sd"> :return: A response to tuning job creation</span>
+<span class="sd"> """</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">check_tuning_config</span><span class="p">(</span><span class="n">config</span><span class="p">)</span>
+
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">create_hyper_parameter_tuning_job</span><span class="p">(</span><span class="o">**</span><span class="n">config</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">wait_for_completion</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">check_status</span><span class="p">(</span><span class="n">config</span><span class="p">[</span><span class="s1">'HyperParameterTuningJobName'</span><span class="p">],</span>
+ <span class="s1">'HyperParameterTuningJobStatus'</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">describe_tuning_job</span><span class="p">,</span>
+ <span class="n">check_interval</span><span class="p">,</span> <span class="n">max_ingestion_time</span>
+ <span class="p">)</span>
+ <span class="k">return</span> <span class="n">response</span></div>
+
+<div class="viewcode-block" id="SageMakerHook.create_transform_job"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.create_transform_job">[docs]</a> <span class="k">def</span> <span class="nf">create_transform_job</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">config</span><span class="p">,</span> <span class="n">wait_for_completion</span><span class="o">=</span><span class="kc" [...]
+ <span class="n">check_interval</span><span class="o">=</span><span class="mi">30</span><span class="p">,</span> <span class="n">max_ingestion_time</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Create a transform job</span>
+
+<span class="sd"> :param config: the config for transform job</span>
+<span class="sd"> :type config: dict</span>
+<span class="sd"> :param wait_for_completion: if the program should keep running until job finishes</span>
+<span class="sd"> :type wait_for_completion: bool</span>
+<span class="sd"> :param check_interval: the time interval in seconds which the operator</span>
+<span class="sd"> will check the status of any SageMaker job</span>
+<span class="sd"> :type check_interval: int</span>
+<span class="sd"> :param max_ingestion_time: the maximum ingestion time in seconds. Any</span>
+<span class="sd"> SageMaker jobs that run longer than this will fail. Setting this to</span>
+<span class="sd"> None implies no timeout for any SageMaker job.</span>
+<span class="sd"> :type max_ingestion_time: int</span>
+<span class="sd"> :return: A response to transform job creation</span>
+<span class="sd"> """</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">check_s3_url</span><span class="p">(</span><span class="n">config</span><span class="p">[</span><span class="s1">'TransformInput'</span><span class="p">][</span><span class="s1">'DataSource'</span><span class="p">][</span><span class="s1">'S3DataSource'</span><span class="p">][</span><span class="s1">'S3Uri'</span><span class="p">])</span>
+
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">create_transform_job</span><span class="p">(</span><span class="o">**</span><span class="n">config</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">wait_for_completion</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">check_status</span><span class="p">(</span><span class="n">config</span><span class="p">[</span><span class="s1">'TransformJobName'</span><span class="p">],</span>
+ <span class="s1">'TransformJobStatus'</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">describe_transform_job</span><span class="p">,</span>
+ <span class="n">check_interval</span><span class="p">,</span> <span class="n">max_ingestion_time</span>
+ <span class="p">)</span>
+ <span class="k">return</span> <span class="n">response</span></div>
+
+<div class="viewcode-block" id="SageMakerHook.create_model"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.create_model">[docs]</a> <span class="k">def</span> <span class="nf">create_model</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">config</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Create a model job</span>
+
+<span class="sd"> :param config: the config for model</span>
+<span class="sd"> :type config: dict</span>
+<span class="sd"> :return: A response to model creation</span>
+<span class="sd"> """</span>
+
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">create_model</span><span class="p">(</span><span class="o">**</span><span class="n">config</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="SageMakerHook.create_endpoint_config"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.create_endpoint_config">[docs]</a> <span class="k">def</span> <span class="nf">create_endpoint_config</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">config</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Create an endpoint config</span>
+
+<span class="sd"> :param config: the config for endpoint-config</span>
+<span class="sd"> :type config: dict</span>
+<span class="sd"> :return: A response to endpoint config creation</span>
+<span class="sd"> """</span>
+
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">create_endpoint_config</span><span class="p">(</span><span class="o">**</span><span class="n">config</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="SageMakerHook.create_endpoint"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.create_endpoint">[docs]</a> <span class="k">def</span> <span class="nf">create_endpoint</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">config</span><span class="p">,</span> <span class="n">wait_for_completion</span><span class="o">=</span><span class="kc">True</span><sp [...]
+ <span class="n">check_interval</span><span class="o">=</span><span class="mi">30</span><span class="p">,</span> <span class="n">max_ingestion_time</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Create an endpoint</span>
+
+<span class="sd"> :param config: the config for endpoint</span>
+<span class="sd"> :type config: dict</span>
+<span class="sd"> :param wait_for_completion: if the program should keep running until job finishes</span>
+<span class="sd"> :type wait_for_completion: bool</span>
+<span class="sd"> :param check_interval: the time interval in seconds which the operator</span>
+<span class="sd"> will check the status of any SageMaker job</span>
+<span class="sd"> :type check_interval: int</span>
+<span class="sd"> :param max_ingestion_time: the maximum ingestion time in seconds. Any</span>
+<span class="sd"> SageMaker jobs that run longer than this will fail. Setting this to</span>
+<span class="sd"> None implies no timeout for any SageMaker job.</span>
+<span class="sd"> :type max_ingestion_time: int</span>
+<span class="sd"> :return: A response to endpoint creation</span>
+<span class="sd"> """</span>
+
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">create_endpoint</span><span class="p">(</span><span class="o">**</span><span class="n">config</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">wait_for_completion</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">check_status</span><span class="p">(</span><span class="n">config</span><span class="p">[</span><span class="s1">'EndpointName'</span><span class="p">],</span>
+ <span class="s1">'EndpointStatus'</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">describe_endpoint</span><span class="p">,</span>
+ <span class="n">check_interval</span><span class="p">,</span> <span class="n">max_ingestion_time</span><span class="p">,</span>
+ <span class="n">non_terminal_states</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">endpoint_non_terminal_states</span>
+ <span class="p">)</span>
+ <span class="k">return</span> <span class="n">response</span></div>
+
+<div class="viewcode-block" id="SageMakerHook.update_endpoint"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.update_endpoint">[docs]</a> <span class="k">def</span> <span class="nf">update_endpoint</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">config</span><span class="p">,</span> <span class="n">wait_for_completion</span><span class="o">=</span><span class="kc">True</span><sp [...]
+ <span class="n">check_interval</span><span class="o">=</span><span class="mi">30</span><span class="p">,</span> <span class="n">max_ingestion_time</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Update an endpoint</span>
+
+<span class="sd"> :param config: the config for endpoint</span>
+<span class="sd"> :type config: dict</span>
+<span class="sd"> :param wait_for_completion: if the program should keep running until job finishes</span>
+<span class="sd"> :type wait_for_completion: bool</span>
+<span class="sd"> :param check_interval: the time interval in seconds which the operator</span>
+<span class="sd"> will check the status of any SageMaker job</span>
+<span class="sd"> :type check_interval: int</span>
+<span class="sd"> :param max_ingestion_time: the maximum ingestion time in seconds. Any</span>
+<span class="sd"> SageMaker jobs that run longer than this will fail. Setting this to</span>
+<span class="sd"> None implies no timeout for any SageMaker job.</span>
+<span class="sd"> :type max_ingestion_time: int</span>
+<span class="sd"> :return: A response to endpoint update</span>
+<span class="sd"> """</span>
+
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">update_endpoint</span><span class="p">(</span><span class="o">**</span><span class="n">config</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">wait_for_completion</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">check_status</span><span class="p">(</span><span class="n">config</span><span class="p">[</span><span class="s1">'EndpointName'</span><span class="p">],</span>
+ <span class="s1">'EndpointStatus'</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">describe_endpoint</span><span class="p">,</span>
+ <span class="n">check_interval</span><span class="p">,</span> <span class="n">max_ingestion_time</span><span class="p">,</span>
+ <span class="n">non_terminal_states</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">endpoint_non_terminal_states</span>
+ <span class="p">)</span>
+ <span class="k">return</span> <span class="n">response</span></div>
+
+<div class="viewcode-block" id="SageMakerHook.describe_training_job"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.describe_training_job">[docs]</a> <span class="k">def</span> <span class="nf">describe_training_job</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Return the training job info associated with the name</span>
+
+<span class="sd"> :param name: the name of the training job</span>
+<span class="sd"> :type name: str</span>
+<span class="sd"> :return: A dict contains all the training job info</span>
+<span class="sd"> """</span>
+
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">describe_training_job</span><span class="p">(</span><span class="n">TrainingJobName</span><span class="o">=</span><span class="n">name</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="SageMakerHook.describe_training_job_with_log"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.describe_training_job_with_log">[docs]</a> <span class="k">def</span> <span class="nf">describe_training_job_with_log</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">job_name</span><span class="p">,</span> <span class="n">positions</span><span class="p">,< [...]
+ <span class="n">instance_count</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">last_description</span><span class="p">,</span>
+ <span class="n">last_describe_job_call</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Return the training job info associated with job_name and print CloudWatch logs</span>
+<span class="sd"> """</span>
+ <span class="n">log_group</span> <span class="o">=</span> <span class="s1">'/aws/sagemaker/TrainingJobs'</span>
+
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">stream_names</span><span class="p">)</span> <span class="o"><</span> <span class="n">instance_count</span><span class="p">:</span>
+ <span class="c1"># Log streams are created whenever a container starts writing to stdout/err, so this list</span>
+ <span class="c1"># may be dynamic until we have a stream for every instance.</span>
+ <span class="n">logs_conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_log_conn</span><span class="p">()</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">streams</span> <span class="o">=</span> <span class="n">logs_conn</span><span class="o">.</span><span class="n">describe_log_streams</span><span class="p">(</span>
+ <span class="n">logGroupName</span><span class="o">=</span><span class="n">log_group</span><span class="p">,</span>
+ <span class="n">logStreamNamePrefix</span><span class="o">=</span><span class="n">job_name</span> <span class="o">+</span> <span class="s1">'/'</span><span class="p">,</span>
+ <span class="n">orderBy</span><span class="o">=</span><span class="s1">'LogStreamName'</span><span class="p">,</span>
+ <span class="n">limit</span><span class="o">=</span><span class="n">instance_count</span>
+ <span class="p">)</span>
+ <span class="n">stream_names</span> <span class="o">=</span> <span class="p">[</span><span class="n">s</span><span class="p">[</span><span class="s1">'logStreamName'</span><span class="p">]</span> <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">streams</span><span class="p">[</span><span class="s1">'logStreams'</span><span class="p">]]</span>
+ <span class="n">positions</span><span class="o">.</span><span class="n">update</span><span class="p">([(</span><span class="n">s</span><span class="p">,</span> <span class="n">Position</span><span class="p">(</span><span class="n">timestamp</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">skip</span><span class="o">=</span><span class="mi">0</span><span class="p">))</span>
+ <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">stream_names</span> <span class="k">if</span> <span class="n">s</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">positions</span><span class="p">])</span>
+ <span class="k">except</span> <span class="n">logs_conn</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">ResourceNotFoundException</span><span class="p">:</span>
+ <span class="c1"># On the very first training job run on an account, there's no log group until</span>
+ <span class="c1"># the container starts logging, so ignore any errors thrown about that</span>
+ <span class="k">pass</span>
+
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">stream_names</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
+ <span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">event</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">multi_stream_iter</span><span class="p">(</span><span class="n">log_group</span><span class="p">,</span> <span class="n">stream_names</span><span class="p">,</span> <span class="n">positions</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">event</span><span class="p">[</span><span class="s1">'message'</span><span class="p">])</span>
+ <span class="n">ts</span><span class="p">,</span> <span class="n">count</span> <span class="o">=</span> <span class="n">positions</span><span class="p">[</span><span class="n">stream_names</span><span class="p">[</span><span class="n">idx</span><span class="p">]]</span>
+ <span class="k">if</span> <span class="n">event</span><span class="p">[</span><span class="s1">'timestamp'</span><span class="p">]</span> <span class="o">==</span> <span class="n">ts</span><span class="p">:</span>
+ <span class="n">positions</span><span class="p">[</span><span class="n">stream_names</span><span class="p">[</span><span class="n">idx</span><span class="p">]]</span> <span class="o">=</span> <span class="n">Position</span><span class="p">(</span><span class="n">timestamp</span><span class="o">=</span><span class="n">ts</span><span class="p">,</span> <span class="n">skip</span><span class="o">=</span><span class="n">count</span> <span class="o">+</span> <span class="m [...]
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">positions</span><span class="p">[</span><span class="n">stream_names</span><span class="p">[</span><span class="n">idx</span><span class="p">]]</span> <span class="o">=</span> <span class="n">Position</span><span class="p">(</span><span class="n">timestamp</span><span class="o">=</span><span class="n">event</span><span class="p">[</span><span class="s1">'timestamp'</span><span class="p">],</span> <span class="n">skip</span><span class="o">=</sp [...]
+
+ <span class="k">if</span> <span class="n">state</span> <span class="o">==</span> <span class="n">LogState</span><span class="o">.</span><span class="n">COMPLETE</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">state</span><span class="p">,</span> <span class="n">last_description</span><span class="p">,</span> <span class="n">last_describe_job_call</span>
+
+ <span class="k">if</span> <span class="n">state</span> <span class="o">==</span> <span class="n">LogState</span><span class="o">.</span><span class="n">JOB_COMPLETE</span><span class="p">:</span>
+ <span class="n">state</span> <span class="o">=</span> <span class="n">LogState</span><span class="o">.</span><span class="n">COMPLETE</span>
+ <span class="k">elif</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span> <span class="o">-</span> <span class="n">last_describe_job_call</span> <span class="o">>=</span> <span class="mi">30</span><span class="p">:</span>
+ <span class="n">description</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">describe_training_job</span><span class="p">(</span><span class="n">job_name</span><span class="p">)</span>
+ <span class="n">last_describe_job_call</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
+
+ <span class="k">if</span> <span class="n">secondary_training_status_changed</span><span class="p">(</span><span class="n">description</span><span class="p">,</span> <span class="n">last_description</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">secondary_training_status_message</span><span class="p">(</span><span class="n">description</span><span class="p">,</span> <span class="n">last_description</span><span class="p">))</span>
+ <span class="n">last_description</span> <span class="o">=</span> <span class="n">description</span>
+
+ <span class="n">status</span> <span class="o">=</span> <span class="n">description</span><span class="p">[</span><span class="s1">'TrainingJobStatus'</span><span class="p">]</span>
+
+ <span class="k">if</span> <span class="n">status</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">non_terminal_states</span><span class="p">:</span>
+ <span class="n">state</span> <span class="o">=</span> <span class="n">LogState</span><span class="o">.</span><span class="n">JOB_COMPLETE</span>
+ <span class="k">return</span> <span class="n">state</span><span class="p">,</span> <span class="n">last_description</span><span class="p">,</span> <span class="n">last_describe_job_call</span></div>
+
+<div class="viewcode-block" id="SageMakerHook.describe_tuning_job"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.describe_tuning_job">[docs]</a> <span class="k">def</span> <span class="nf">describe_tuning_job</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Return the tuning job info associated with the name</span>
+
+<span class="sd"> :param name: the name of the tuning job</span>
+<span class="sd"> :type name: string</span>
+<span class="sd"> :return: A dict contains all the tuning job info</span>
+<span class="sd"> """</span>
+
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">describe_hyper_parameter_tuning_job</span><span class="p">(</span><span class="n">HyperParameterTuningJobName</span><span class="o">=</span><span class="n">name</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="SageMakerHook.describe_model"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.describe_model">[docs]</a> <span class="k">def</span> <span class="nf">describe_model</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Return the SageMaker model info associated with the name</span>
+
+<span class="sd"> :param name: the name of the SageMaker model</span>
+<span class="sd"> :type name: string</span>
+<span class="sd"> :return: A dict contains all the model info</span>
+<span class="sd"> """</span>
+
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">describe_model</span><span class="p">(</span><span class="n">ModelName</span><span class="o">=</span><span class="n">name</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="SageMakerHook.describe_transform_job"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.describe_transform_job">[docs]</a> <span class="k">def</span> <span class="nf">describe_transform_job</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Return the transform job info associated with the name</span>
+
+<span class="sd"> :param name: the name of the transform job</span>
+<span class="sd"> :type name: string</span>
+<span class="sd"> :return: A dict contains all the transform job info</span>
+<span class="sd"> """</span>
+
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">describe_transform_job</span><span class="p">(</span><span class="n">TransformJobName</span><span class="o">=</span><span class="n">name</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="SageMakerHook.describe_endpoint_config"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.describe_endpoint_config">[docs]</a> <span class="k">def</span> <span class="nf">describe_endpoint_config</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Return the endpoint config info associated with the name</span>
+
+<span class="sd"> :param name: the name of the endpoint config</span>
+<span class="sd"> :type name: string</span>
+<span class="sd"> :return: A dict contains all the endpoint config info</span>
+<span class="sd"> """</span>
+
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">describe_endpoint_config</span><span class="p">(</span><span class="n">EndpointConfigName</span><span class="o">=</span><span class="n">name</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="SageMakerHook.describe_endpoint"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.describe_endpoint">[docs]</a> <span class="k">def</span> <span class="nf">describe_endpoint</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> :param name: the name of the endpoint</span>
+<span class="sd"> :type name: string</span>
+<span class="sd"> :return: A dict contains all the endpoint info</span>
+<span class="sd"> """</span>
+
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">describe_endpoint</span><span class="p">(</span><span class="n">EndpointName</span><span class="o">=</span><span class="n">name</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="SageMakerHook.check_status"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.check_status">[docs]</a> <span class="k">def</span> <span class="nf">check_status</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">job_name</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span>
+ <span class="n">describe_function</span><span class="p">,</span> <span class="n">check_interval</span><span class="p">,</span>
+ <span class="n">max_ingestion_time</span><span class="p">,</span>
+ <span class="n">non_terminal_states</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Check status of a SageMaker job</span>
+
+<span class="sd"> :param job_name: name of the job to check status</span>
+<span class="sd"> :type job_name: str</span>
+<span class="sd"> :param key: the key of the response dict</span>
+<span class="sd"> that points to the state</span>
+<span class="sd"> :type key: str</span>
+<span class="sd"> :param describe_function: the function used to retrieve the status</span>
+<span class="sd"> :type describe_function: python callable</span>
+<span class="sd"> :param args: the arguments for the function</span>
+<span class="sd"> :param check_interval: the time interval in seconds which the operator</span>
+<span class="sd"> will check the status of any SageMaker job</span>
+<span class="sd"> :type check_interval: int</span>
+<span class="sd"> :param max_ingestion_time: the maximum ingestion time in seconds. Any</span>
+<span class="sd"> SageMaker jobs that run longer than this will fail. Setting this to</span>
+<span class="sd"> None implies no timeout for any SageMaker job.</span>
+<span class="sd"> :type max_ingestion_time: int</span>
+<span class="sd"> :param non_terminal_states: the set of nonterminal states</span>
+<span class="sd"> :type non_terminal_states: set</span>
+<span class="sd"> :return: response of describe call after job is done</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">non_terminal_states</span><span class="p">:</span>
+ <span class="n">non_terminal_states</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">non_terminal_states</span>
+
+ <span class="n">sec</span> <span class="o">=</span> <span class="mi">0</span>
+ <span class="n">running</span> <span class="o">=</span> <span class="kc">True</span>
+
+ <span class="k">while</span> <span class="n">running</span><span class="p">:</span>
+ <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="n">check_interval</span><span class="p">)</span>
+ <span class="n">sec</span> <span class="o">=</span> <span class="n">sec</span> <span class="o">+</span> <span class="n">check_interval</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="n">describe_function</span><span class="p">(</span><span class="n">job_name</span><span class="p">)</span>
+ <span class="n">status</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Job still running for </span><span class="si">%s</span><span class="s1"> seconds... '</span>
+ <span class="s1">'current status is </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="p">(</span><span class="n">sec</span><span class="p">,</span> <span class="n">status</span><span class="p">))</span>
+ <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Could not get status of the SageMaker job'</span><span class="p">)</span>
+ <span class="k">except</span> <span class="n">ClientError</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'AWS request failed, check logs for more info'</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">status</span> <span class="ow">in</span> <span class="n">non_terminal_states</span><span class="p">:</span>
+ <span class="n">running</span> <span class="o">=</span> <span class="kc">True</span>
+ <span class="k">elif</span> <span class="n">status</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">failed_states</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'SageMaker job failed because </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">response</span><span class="p">[</span><span class="s1">'FailureReason'</span><span class="p">])</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">running</span> <span class="o">=</span> <span class="kc">False</span>
+
+ <span class="k">if</span> <span class="n">max_ingestion_time</span> <span class="ow">and</span> <span class="n">sec</span> <span class="o">></span> <span class="n">max_ingestion_time</span><span class="p">:</span>
+ <span class="c1"># ensure that the job gets killed if the max ingestion time is exceeded</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'SageMaker job took more than </span><span class="si">%s</span><span class="s1"> seconds'</span><span class="p">,</span> <span class="n">max_ingestion_time</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'SageMaker Job Compeleted'</span><span class="p">)</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="n">describe_function</span><span class="p">(</span><span class="n">job_name</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">response</span></div>
+
+<div class="viewcode-block" id="SageMakerHook.check_training_status_with_log"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sagemaker_hook.SageMakerHook.check_training_status_with_log">[docs]</a> <span class="k">def</span> <span class="nf">check_training_status_with_log</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">job_name</span><span class="p">,</span> <span class="n">non_terminal_states</span><span cl [...]
+ <span class="n">wait_for_completion</span><span class="p">,</span> <span class="n">check_interval</span><span class="p">,</span> <span class="n">max_ingestion_time</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Display the logs for a given training job, optionally tailing them until the</span>
+<span class="sd"> job is complete.</span>
+
+<span class="sd"> :param job_name: name of the training job to check status and display logs for</span>
+<span class="sd"> :type job_name: str</span>
+<span class="sd"> :param non_terminal_states: the set of non_terminal states</span>
+<span class="sd"> :type non_terminal_states: set</span>
+<span class="sd"> :param failed_states: the set of failed states</span>
+<span class="sd"> :type failed_states: set</span>
+<span class="sd"> :param wait_for_completion: Whether to keep looking for new log entries</span>
+<span class="sd"> until the job completes</span>
+<span class="sd"> :type wait_for_completion: bool</span>
+<span class="sd"> :param check_interval: The interval in seconds between polling for new log entries and job completion</span>
+<span class="sd"> :type check_interval: int</span>
+<span class="sd"> :param max_ingestion_time: the maximum ingestion time in seconds. Any</span>
+<span class="sd"> SageMaker jobs that run longer than this will fail. Setting this to</span>
+<span class="sd"> None implies no timeout for any SageMaker job.</span>
+<span class="sd"> :type max_ingestion_time: int</span>
+<span class="sd"> :return: None</span>
+<span class="sd"> """</span>
+
+ <span class="n">sec</span> <span class="o">=</span> <span class="mi">0</span>
+ <span class="n">description</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">describe_training_job</span><span class="p">(</span><span class="n">job_name</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">secondary_training_status_message</span><span class="p">(</span><span class="n">description</span><span class="p">,</span> <span class="kc">None</span><span class="p">))</span>
+ <span class="n">instance_count</span> <span class="o">=</span> <span class="n">description</span><span class="p">[</span><span class="s1">'ResourceConfig'</span><span class="p">][</span><span class="s1">'InstanceCount'</span><span class="p">]</span>
+ <span class="n">status</span> <span class="o">=</span> <span class="n">description</span><span class="p">[</span><span class="s1">'TrainingJobStatus'</span><span class="p">]</span>
+
+ <span class="n">stream_names</span> <span class="o">=</span> <span class="p">[]</span> <span class="c1"># The list of log streams</span>
+ <span class="n">positions</span> <span class="o">=</span> <span class="p">{}</span> <span class="c1"># The current position in each stream, map of stream name -> position</span>
+
+ <span class="n">job_already_completed</span> <span class="o">=</span> <span class="n">status</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">non_terminal_states</span>
+
+ <span class="n">state</span> <span class="o">=</span> <span class="n">LogState</span><span class="o">.</span><span class="n">TAILING</span> <span class="k">if</span> <span class="n">wait_for_completion</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">job_already_completed</span> <span class="k">else</span> <span class="n">LogState</span><span class="o">.</span><span class="n">COMPLETE</span>
+
+ <span class="c1"># The loop below implements a state machine that alternates between checking the job status and</span>
+ <span class="c1"># reading whatever is available in the logs at this point. Note, that if we were called with</span>
+ <span class="c1"># wait_for_completion == False, we never check the job status.</span>
+ <span class="c1">#</span>
+ <span class="c1"># If wait_for_completion == TRUE and job is not completed, the initial state is TAILING</span>
+ <span class="c1"># If wait_for_completion == FALSE, the initial state is COMPLETE</span>
+ <span class="c1"># (doesn't matter if the job really is complete).</span>
+ <span class="c1">#</span>
+ <span class="c1"># The state table:</span>
+ <span class="c1">#</span>
+ <span class="c1"># STATE ACTIONS CONDITION NEW STATE</span>
+ <span class="c1"># ---------------- ---------------- ----------------- ----------------</span>
+ <span class="c1"># TAILING Read logs, Pause, Get status Job complete JOB_COMPLETE</span>
+ <span class="c1"># Else TAILING</span>
+ <span class="c1"># JOB_COMPLETE Read logs, Pause Any COMPLETE</span>
+ <span class="c1"># COMPLETE Read logs, Exit N/A</span>
+ <span class="c1">#</span>
+ <span class="c1"># Notes:</span>
+ <span class="c1"># - The JOB_COMPLETE state forces us to do an extra pause and read any items that</span>
+ <span class="c1"># got to Cloudwatch after the job was marked complete.</span>
+ <span class="n">last_describe_job_call</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
+ <span class="n">last_description</span> <span class="o">=</span> <span class="n">description</span>
+
+ <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+ <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="n">check_interval</span><span class="p">)</span>
+ <span class="n">sec</span> <span class="o">=</span> <span class="n">sec</span> <span class="o">+</span> <span class="n">check_interval</span>
+
+ <span class="n">state</span><span class="p">,</span> <span class="n">last_description</span><span class="p">,</span> <span class="n">last_describe_job_call</span> <span class="o">=</span> \
+ <span class="bp">self</span><span class="o">.</span><span class="n">describe_training_job_with_log</span><span class="p">(</span><span class="n">job_name</span><span class="p">,</span> <span class="n">positions</span><span class="p">,</span> <span class="n">stream_names</span><span class="p">,</span>
+ <span class="n">instance_count</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">last_description</span><span class="p">,</span>
+ <span class="n">last_describe_job_call</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">state</span> <span class="o">==</span> <span class="n">LogState</span><span class="o">.</span><span class="n">COMPLETE</span><span class="p">:</span>
+ <span class="k">break</span>
+
+ <span class="k">if</span> <span class="n">max_ingestion_time</span> <span class="ow">and</span> <span class="n">sec</span> <span class="o">></span> <span class="n">max_ingestion_time</span><span class="p">:</span>
+ <span class="c1"># ensure that the job gets killed if the max ingestion time is exceeded</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'SageMaker job took more than </span><span class="si">%s</span><span class="s1"> seconds'</span><span class="p">,</span> <span class="n">max_ingestion_time</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">wait_for_completion</span><span class="p">:</span>
+ <span class="n">status</span> <span class="o">=</span> <span class="n">last_description</span><span class="p">[</span><span class="s1">'TrainingJobStatus'</span><span class="p">]</span>
+ <span class="k">if</span> <span class="n">status</span> <span class="ow">in</span> <span class="n">failed_states</span><span class="p">:</span>
+ <span class="n">reason</span> <span class="o">=</span> <span class="n">last_description</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'FailureReason'</span><span class="p">,</span> <span class="s1">'(No reason provided)'</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Error training </span><span class="si">{}</span><span class="s1">: </span><span class="si">{}</span><span class="s1"> Reason: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">job_name</span><span class="p">,</span> <span class="n">status</span><span class= [...]
+ <span class="n">billable_time</span> <span class="o">=</span> <span class="p">(</span><span class="n">last_description</span><span class="p">[</span><span class="s1">'TrainingEndTime'</span><span class="p">]</span> <span class="o">-</span> <span class="n">last_description</span><span class="p">[</span><span class="s1">'TrainingStartTime'</span><span class="p">])</span> \
+ <span class="o">*</span> <span class="n">instance_count</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Billable seconds:</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">billable_time</span><span class="o">.</span><span class="n">total_seconds</span><s [...]
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/hooks/segment_hook.html b/_modules/airflow/contrib/hooks/segment_hook.html
index ccb8fd0..fb4396f 100644
--- a/_modules/airflow/contrib/hooks/segment_hook.html
+++ b/_modules/airflow/contrib/hooks/segment_hook.html
@@ -281,20 +281,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/sftp_hook.html b/_modules/airflow/contrib/hooks/sftp_hook.html
index efc5485..84a9fe8 100644
--- a/_modules/airflow/contrib/hooks/sftp_hook.html
+++ b/_modules/airflow/contrib/hooks/sftp_hook.html
@@ -182,11 +182,14 @@
<span class="kn">import</span> <span class="nn">pysftp</span>
<span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">import</span> <span class="nn">datetime</span>
-<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.ssh_hook</span> <span class="k">import</span> <span class="n">SSHHook</span>
-<div class="viewcode-block" id="SFTPHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sftp_hook.SFTPHook">[docs]</a><span class="k">class</span> <span class="nc">SFTPHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+<div class="viewcode-block" id="SFTPHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sftp_hook.SFTPHook">[docs]</a><span class="k">class</span> <span class="nc">SFTPHook</span><span class="p">(</span><span class="n">SSHHook</span><span class="p">):</span>
<span class="sd">"""</span>
+<span class="sd"> This hook is inherited from SSH hook. Please refer to SSH hook for the input</span>
+<span class="sd"> arguments.</span>
+
<span class="sd"> Interact with SFTP. Aims to be interchangeable with FTPHook.</span>
<span class="sd"> Pitfalls: - In contrast with FTPHook describe_directory only returns size, type and</span>
@@ -200,32 +203,74 @@
<span class="sd"> Errors that may occur throughout but should be handled downstream.</span>
<span class="sd"> """</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ftp_conn_id</span><span class="o">=</span><span class="s1">'sftp_default'</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">ftp_conn_id</span> <span class="o">=</span> <span class="n">ftp_conn_id</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ftp_conn_id</span><span class="o">=</span><span class="s1">'sftp_default'</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="n">kwargs</span><span class="p">[</span><span class="s1">'ssh_conn_id'</span><span class="p">]</span> <span class="o">=</span> <span class="n">ftp_conn_id</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">SFTPHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
<span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">private_key_pass</span> <span class="o">=</span> <span class="kc">None</span>
+
+ <span class="c1"># Fail for unverified hosts, unless this is explicitly allowed</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">no_host_key_check</span> <span class="o">=</span> <span class="kc">False</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">extra_options</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span>
+ <span class="k">if</span> <span class="s1">'private_key_pass'</span> <span class="ow">in</span> <span class="n">extra_options</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">private_key_pass</span> <span class="o">=</span> <span class="n">extra_options</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'private_key_pass'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+
+ <span class="c1"># For backward compatibility</span>
+ <span class="c1"># TODO: remove in Airflow 2.1</span>
+ <span class="kn">import</span> <span class="nn">warnings</span>
+ <span class="k">if</span> <span class="s1">'ignore_hostkey_verification'</span> <span class="ow">in</span> <span class="n">extra_options</span><span class="p">:</span>
+ <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+ <span class="s1">'Extra option `ignore_hostkey_verification` is deprecated.'</span>
+ <span class="s1">'Please use `no_host_key_check` instead.'</span>
+ <span class="s1">'This option will be removed in Airflow 2.1'</span><span class="p">,</span>
+ <span class="ne">DeprecationWarning</span><span class="p">,</span>
+ <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span>
+ <span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">no_host_key_check</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span>
+ <span class="n">extra_options</span><span class="p">[</span><span class="s1">'ignore_hostkey_verification'</span><span class="p">]</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'true'</span>
+
+ <span class="k">if</span> <span class="s1">'no_host_key_check'</span> <span class="ow">in</span> <span class="n">extra_options</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">no_host_key_check</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span>
+ <span class="n">extra_options</span><span class="p">[</span><span class="s1">'no_host_key_check'</span><span class="p">])</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'true'</span>
+
+ <span class="k">if</span> <span class="s1">'private_key'</span> <span class="ow">in</span> <span class="n">extra_options</span><span class="p">:</span>
+ <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+ <span class="s1">'Extra option `private_key` is deprecated.'</span>
+ <span class="s1">'Please use `key_file` instead.'</span>
+ <span class="s1">'This option will be removed in Airflow 2.1'</span><span class="p">,</span>
+ <span class="ne">DeprecationWarning</span><span class="p">,</span>
+ <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span>
+ <span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span> <span class="o">=</span> <span class="n">extra_options</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'private_key'</span><span class="p">)</span>
<div class="viewcode-block" id="SFTPHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sftp_hook.SFTPHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Returns an SFTP connection object</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
- <span class="n">params</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ftp_conn_id</span><span class="p">)</span>
<span class="n">cnopts</span> <span class="o">=</span> <span class="n">pysftp</span><span class="o">.</span><span class="n">CnOpts</span><span class="p">()</span>
- <span class="k">if</span> <span class="p">(</span><span class="s1">'ignore_hostkey_verification'</span> <span class="ow">in</span> <span class="n">params</span><span class="o">.</span><span class="n">extra_dejson</span> <span class="ow">and</span>
- <span class="n">params</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s1">'ignore_hostkey_verification'</span><span class="p">]):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">no_host_key_check</span><span class="p">:</span>
<span class="n">cnopts</span><span class="o">.</span><span class="n">hostkeys</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="n">cnopts</span><span class="o">.</span><span class="n">compression</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">compress</span>
<span class="n">conn_params</span> <span class="o">=</span> <span class="p">{</span>
- <span class="s1">'host'</span><span class="p">:</span> <span class="n">params</span><span class="o">.</span><span class="n">host</span><span class="p">,</span>
- <span class="s1">'port'</span><span class="p">:</span> <span class="n">params</span><span class="o">.</span><span class="n">port</span><span class="p">,</span>
- <span class="s1">'username'</span><span class="p">:</span> <span class="n">params</span><span class="o">.</span><span class="n">login</span><span class="p">,</span>
+ <span class="s1">'host'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span>
+ <span class="s1">'port'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">port</span><span class="p">,</span>
+ <span class="s1">'username'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">,</span>
<span class="s1">'cnopts'</span><span class="p">:</span> <span class="n">cnopts</span>
<span class="p">}</span>
- <span class="k">if</span> <span class="n">params</span><span class="o">.</span><span class="n">password</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
- <span class="n">conn_params</span><span class="p">[</span><span class="s1">'password'</span><span class="p">]</span> <span class="o">=</span> <span class="n">params</span><span class="o">.</span><span class="n">password</span>
- <span class="k">if</span> <span class="s1">'private_key'</span> <span class="ow">in</span> <span class="n">params</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">:</span>
- <span class="n">conn_params</span><span class="p">[</span><span class="s1">'private_key'</span><span class="p">]</span> <span class="o">=</span> <span class="n">params</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s1">'private_key'</span><span class="p">]</span>
- <span class="k">if</span> <span class="s1">'private_key_pass'</span> <span class="ow">in</span> <span class="n">params</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">:</span>
- <span class="n">conn_params</span><span class="p">[</span><span class="s1">'private_key_pass'</span><span class="p">]</span> <span class="o">=</span> <span class="n">params</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s1">'private_key_pass'</span><span class="p">]</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span><span class="o">.</span><span class="n">strip</span><span class="p">():</span>
+ <span class="n">conn_params</span><span class="p">[</span><span class="s1">'password'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span><span class="p">:</span>
+ <span class="n">conn_params</span><span class="p">[</span><span class="s1">'private_key'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">private_key_pass</span><span class="p">:</span>
+ <span class="n">conn_params</span><span class="p">[</span><span class="s1">'private_key_pass'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">private_key_pass</span>
+
<span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="n">pysftp</span><span class="o">.</span><span class="n">Connection</span><span class="p">(</span><span class="o">**</span><span class="n">conn_params</span><span class="p">)</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span></div>
@@ -358,20 +403,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/slack_webhook_hook.html b/_modules/airflow/contrib/hooks/slack_webhook_hook.html
index 8112c68..4ac5d9b 100644
--- a/_modules/airflow/contrib/hooks/slack_webhook_hook.html
+++ b/_modules/airflow/contrib/hooks/slack_webhook_hook.html
@@ -318,20 +318,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/snowflake_hook.html b/_modules/airflow/contrib/hooks/snowflake_hook.html
index aa15463..757efeb 100644
--- a/_modules/airflow/contrib/hooks/snowflake_hook.html
+++ b/_modules/airflow/contrib/hooks/snowflake_hook.html
@@ -287,20 +287,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/spark_jdbc_hook.html b/_modules/airflow/contrib/hooks/spark_jdbc_hook.html
index f6fe261..1af0775 100644
--- a/_modules/airflow/contrib/hooks/spark_jdbc_hook.html
+++ b/_modules/airflow/contrib/hooks/spark_jdbc_hook.html
@@ -435,20 +435,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/spark_sql_hook.html b/_modules/airflow/contrib/hooks/spark_sql_hook.html
index ba78c7b..95e9e4b 100644
--- a/_modules/airflow/contrib/hooks/spark_sql_hook.html
+++ b/_modules/airflow/contrib/hooks/spark_sql_hook.html
@@ -351,20 +351,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/spark_submit_hook.html b/_modules/airflow/contrib/hooks/spark_submit_hook.html
index 9e47eef..db35e88 100644
--- a/_modules/airflow/contrib/hooks/spark_submit_hook.html
+++ b/_modules/airflow/contrib/hooks/spark_submit_hook.html
@@ -756,20 +756,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/sqoop_hook.html b/_modules/airflow/contrib/hooks/sqoop_hook.html
index 9c13d5f..b6acb04 100644
--- a/_modules/airflow/contrib/hooks/sqoop_hook.html
+++ b/_modules/airflow/contrib/hooks/sqoop_hook.html
@@ -541,20 +541,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/ssh_hook.html b/_modules/airflow/contrib/hooks/ssh_hook.html
index 453721f..3123f1d 100644
--- a/_modules/airflow/contrib/hooks/ssh_hook.html
+++ b/_modules/airflow/contrib/hooks/ssh_hook.html
@@ -180,11 +180,12 @@
<span class="kn">import</span> <span class="nn">getpass</span>
<span class="kn">import</span> <span class="nn">os</span>
+<span class="kn">import</span> <span class="nn">warnings</span>
<span class="kn">import</span> <span class="nn">paramiko</span>
<span class="kn">from</span> <span class="nn">paramiko.config</span> <span class="k">import</span> <span class="n">SSH_PORT</span>
+<span class="kn">from</span> <span class="nn">sshtunnel</span> <span class="k">import</span> <span class="n">SSHTunnelForwarder</span>
-<span class="kn">from</span> <span class="nn">contextlib</span> <span class="k">import</span> <span class="n">contextmanager</span>
<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
@@ -223,7 +224,7 @@
<span class="n">username</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">password</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">key_file</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">port</span><span class="o">=</span><span class="n">SSH_PORT</span><span class="p">,</span>
+ <span class="n">port</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">timeout</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span>
<span class="n">keepalive_interval</span><span class="o">=</span><span class="mi">30</span>
<span class="p">):</span>
@@ -233,165 +234,170 @@
<span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="o">=</span> <span class="n">username</span>
<span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="o">=</span> <span class="n">password</span>
<span class="bp">self</span><span class="o">.</span><span class="n">key_file</span> <span class="o">=</span> <span class="n">key_file</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">port</span> <span class="o">=</span> <span class="n">port</span>
<span class="bp">self</span><span class="o">.</span><span class="n">timeout</span> <span class="o">=</span> <span class="n">timeout</span>
<span class="bp">self</span><span class="o">.</span><span class="n">keepalive_interval</span> <span class="o">=</span> <span class="n">keepalive_interval</span>
+
<span class="c1"># Default values, overridable from Connection</span>
<span class="bp">self</span><span class="o">.</span><span class="n">compress</span> <span class="o">=</span> <span class="kc">True</span>
<span class="bp">self</span><span class="o">.</span><span class="n">no_host_key_check</span> <span class="o">=</span> <span class="kc">True</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">host_proxy</span> <span class="o">=</span> <span class="kc">None</span>
+
+ <span class="c1"># Placeholder for deprecated __enter__</span>
<span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="kc">None</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">port</span> <span class="o">=</span> <span class="n">port</span>
- <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'Creating SSH client for conn_id: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span><span class="p">)</span>
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
- <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span><span class="p">)</span>
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span>
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span>
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span>
- <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">port</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span>
- <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
- <span class="n">extra_options</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span> <span class="o">=</span> <span class="n">extra_options</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"key_file"</span><span class="p">)</span>
-
- <span class="k">if</span> <span class="s2">"timeout"</span> <span class="ow">in</span> <span class="n">extra_options</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">extra_options</span><span class="p">[</span><span class="s2">"timeout"</span><span class="p">],</span> <span class="mi">10</span><span class="p">)</span>
-
- <span class="k">if</span> <span class="s2">"compress"</span> <span class="ow">in</span> <span class="n">extra_options</span> \
- <span class="ow">and</span> <span class="nb">str</span><span class="p">(</span><span class="n">extra_options</span><span class="p">[</span><span class="s2">"compress"</span><span class="p">])</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'false'</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">compress</span> <span class="o">=</span> <span class="kc">False</span>
- <span class="k">if</span> <span class="s2">"no_host_key_check"</span> <span class="ow">in</span> <span class="n">extra_options</span> \
- <span class="ow">and</span> \
- <span class="nb">str</span><span class="p">(</span><span class="n">extra_options</span><span class="p">[</span><span class="s2">"no_host_key_check"</span><span class="p">])</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'false'</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">no_host_key_check</span> <span class="o">=</span> <span class="kc">False</span>
-
- <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">:</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Missing required param: remote_host"</span><span class="p">)</span>
-
- <span class="c1"># Auto detecting username values from system</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
- <span class="s2">"username to ssh to host: </span><span class="si">%s</span><span class="s2"> is not specified for connection id"</span>
- <span class="s2">" </span><span class="si">%s</span><span class="s2">. Using system's default provided by getpass.getuser()"</span><span class="p">,</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span>
- <span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="o">=</span> <span class="n">getpass</span><span class="o">.</span><span class="n">getuser</span><span class="p">()</span>
-
- <span class="n">host_proxy</span> <span class="o">=</span> <span class="kc">None</span>
- <span class="n">user_ssh_config_filename</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">expanduser</span><span class="p">(</span><span class="s1">'~/.ssh/config'</span><span class="p">)</span>
- <span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">user_ssh_config_filename</span><span class="p">):</span>
- <span class="n">ssh_conf</span> <span class="o">=</span> <span class="n">paramiko</span><span class="o">.</span><span class="n">SSHConfig</span><span class="p">()</span>
- <span class="n">ssh_conf</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="nb">open</span><span class="p">(</span><span class="n">user_ssh_config_filename</span><span class="p">))</span>
- <span class="n">host_info</span> <span class="o">=</span> <span class="n">ssh_conf</span><span class="o">.</span><span class="n">lookup</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">)</span>
- <span class="k">if</span> <span class="n">host_info</span> <span class="ow">and</span> <span class="n">host_info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'proxycommand'</span><span class="p">):</span>
- <span class="n">host_proxy</span> <span class="o">=</span> <span class="n">paramiko</span><span class="o">.</span><span class="n">ProxyCommand</span><span class="p">(</span><span class="n">host_info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'proxycommand'</span><span class="p">))</span>
-
- <span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span><span class="p">):</span>
- <span class="k">if</span> <span class="n">host_info</span> <span class="ow">and</span> <span class="n">host_info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'identityfile'</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span> <span class="o">=</span> <span class="n">host_info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'identityfile'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
-
- <span class="k">try</span><span class="p">:</span>
- <span class="n">client</span> <span class="o">=</span> <span class="n">paramiko</span><span class="o">.</span><span class="n">SSHClient</span><span class="p">()</span>
- <span class="n">client</span><span class="o">.</span><span class="n">load_system_host_keys</span><span class="p">()</span>
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">no_host_key_check</span><span class="p">:</span>
- <span class="c1"># Default is RejectPolicy</span>
- <span class="n">client</span><span class="o">.</span><span class="n">set_missing_host_key_policy</span><span class="p">(</span><span class="n">paramiko</span><span class="o">.</span><span class="n">AutoAddPolicy</span><span class="p">())</span>
-
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span><span class="o">.</span><span class="n">strip</span><span class="p">():</span>
- <span class="n">client</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">hostname</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span>
- <span class="n">username</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">,</span>
- <span class="n">password</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">password</span><span class="p">,</span>
- <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">,</span>
- <span class="n">compress</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">compress</span><span class="p">,</span>
- <span class="n">port</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">port</span><span class="p">,</span>
- <span class="n">sock</span><span class="o">=</span><span class="n">host_proxy</span><span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">client</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">hostname</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span>
- <span class="n">username</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">,</span>
- <span class="n">key_filename</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">key_file</span><span class="p">,</span>
- <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">,</span>
- <span class="n">compress</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">compress</span><span class="p">,</span>
- <span class="n">port</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">port</span><span class="p">,</span>
- <span class="n">sock</span><span class="o">=</span><span class="n">host_proxy</span><span class="p">)</span>
-
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">keepalive_interval</span><span class="p">:</span>
- <span class="n">client</span><span class="o">.</span><span class="n">get_transport</span><span class="p">()</span><span class="o">.</span><span class="n">set_keepalive</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">keepalive_interval</span><span class="p">)</span>
-
- <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="n">client</span>
- <span class="k">except</span> <span class="n">paramiko</span><span class="o">.</span><span class="n">AuthenticationException</span> <span class="k">as</span> <span class="n">auth_error</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
- <span class="s2">"Auth failed while connecting to host: </span><span class="si">%s</span><span class="s2">, error: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span> <span class="n">auth_error</span>
- <span class="p">)</span>
- <span class="k">except</span> <span class="n">paramiko</span><span class="o">.</span><span class="n">SSHException</span> <span class="k">as</span> <span class="n">ssh_error</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
- <span class="s2">"Failed connecting to host: </span><span class="si">%s</span><span class="s2">, error: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span> <span class="n">ssh_error</span>
- <span class="p">)</span>
- <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
- <span class="s2">"Error connecting to host: </span><span class="si">%s</span><span class="s2">, error: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span> <span class="n">error</span>
- <span class="p">)</span>
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span>
-
-<div class="viewcode-block" id="SSHHook.create_tunnel"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ssh_hook.SSHHook.create_tunnel">[docs]</a> <span class="nd">@contextmanager</span>
- <span class="k">def</span> <span class="nf">create_tunnel</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">local_port</span><span class="p">,</span> <span class="n">remote_port</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">remote_host</span><span class="o">=</span><span class="s2">"localhost"</span><span class="p">):</span>
+ <span class="c1"># Use connection to override defaults</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">port</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">port</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span>
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">extra_options</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span> <span class="o">=</span> <span class="n">extra_options</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"key_file"</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="s2">"timeout"</span> <span class="ow">in</span> <span class="n">extra_options</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">extra_options</span><span class="p">[</span><span class="s2">"timeout"</span><span class="p">],</span> <span class="mi">10</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="s2">"compress"</span> <span class="ow">in</span> <span class="n">extra_options</span>\
+ <span class="ow">and</span> <span class="nb">str</span><span class="p">(</span><span class="n">extra_options</span><span class="p">[</span><span class="s2">"compress"</span><span class="p">])</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'false'</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">compress</span> <span class="o">=</span> <span class="kc">False</span>
+ <span class="k">if</span> <span class="s2">"no_host_key_check"</span> <span class="ow">in</span> <span class="n">extra_options</span>\
+ <span class="ow">and</span>\
+ <span class="nb">str</span><span class="p">(</span><span class="n">extra_options</span><span class="p">[</span><span class="s2">"no_host_key_check"</span><span class="p">])</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'false'</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">no_host_key_check</span> <span class="o">=</span> <span class="kc">False</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Missing required param: remote_host"</span><span class="p">)</span>
+
+ <span class="c1"># Auto detecting username values from system</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
+ <span class="s2">"username to ssh to host: </span><span class="si">%s</span><span class="s2"> is not specified for connection id"</span>
+ <span class="s2">" </span><span class="si">%s</span><span class="s2">. Using system's default provided by getpass.getuser()"</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span>
+ <span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="o">=</span> <span class="n">getpass</span><span class="o">.</span><span class="n">getuser</span><span class="p">()</span>
+
+ <span class="n">user_ssh_config_filename</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">expanduser</span><span class="p">(</span><span class="s1">'~/.ssh/config'</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">user_ssh_config_filename</span><span class="p">):</span>
+ <span class="n">ssh_conf</span> <span class="o">=</span> <span class="n">paramiko</span><span class="o">.</span><span class="n">SSHConfig</span><span class="p">()</span>
+ <span class="n">ssh_conf</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="nb">open</span><span class="p">(</span><span class="n">user_ssh_config_filename</span><span class="p">))</span>
+ <span class="n">host_info</span> <span class="o">=</span> <span class="n">ssh_conf</span><span class="o">.</span><span class="n">lookup</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">host_info</span> <span class="ow">and</span> <span class="n">host_info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'proxycommand'</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">host_proxy</span> <span class="o">=</span> <span class="n">paramiko</span><span class="o">.</span><span class="n">ProxyCommand</span><span class="p">(</span><span class="n">host_info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'proxycommand'</span><span class="p">))</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">host_info</span> <span class="ow">and</span> <span class="n">host_info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'identityfile'</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span> <span class="o">=</span> <span class="n">host_info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'identityfile'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">port</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">port</span> <span class="ow">or</span> <span class="n">SSH_PORT</span>
+
+<div class="viewcode-block" id="SSHHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ssh_hook.SSHHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
-<span class="sd"> Creates a tunnel between two hosts. Like ssh -L <LOCAL_PORT>:host:<REMOTE_PORT>.</span>
-<span class="sd"> Remember to close() the returned "tunnel" object in order to clean up</span>
-<span class="sd"> after yourself when you are done with the tunnel.</span>
+<span class="sd"> Opens a ssh connection to the remote host.</span>
-<span class="sd"> :param local_port:</span>
-<span class="sd"> :type local_port: int</span>
-<span class="sd"> :param remote_port:</span>
-<span class="sd"> :type remote_port: int</span>
-<span class="sd"> :param remote_host:</span>
-<span class="sd"> :type remote_host: str</span>
-<span class="sd"> :return:</span>
+<span class="sd"> :return paramiko.SSHClient object</span>
<span class="sd"> """</span>
- <span class="kn">import</span> <span class="nn">subprocess</span>
- <span class="c1"># this will ensure the connection to the ssh.remote_host from where the tunnel</span>
- <span class="c1"># is getting created</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
-
- <span class="n">tunnel_host</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">:</span><span class="si">{1}</span><span class="s2">:</span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">local_port</span><span class="p">,</span> <span class="n">remote_host</span><span class="p">,</span> <span class="n">remote_port</span><span [...]
-
- <span class="n">ssh_cmd</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"ssh"</span><span class="p">,</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">@</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">,</span> <span class="bp" [...]
- <span class="s2">"-o"</span><span class="p">,</span> <span class="s2">"ControlMaster=no"</span><span class="p">,</span>
- <span class="s2">"-o"</span><span class="p">,</span> <span class="s2">"UserKnownHostsFile=/dev/null"</span><span class="p">,</span>
- <span class="s2">"-o"</span><span class="p">,</span> <span class="s2">"StrictHostKeyChecking=no"</span><span class="p">]</span>
-
- <span class="n">ssh_tunnel_cmd</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"-L"</span><span class="p">,</span> <span class="n">tunnel_host</span><span class="p">,</span>
- <span class="s2">"echo -n ready && cat"</span>
- <span class="p">]</span>
-
- <span class="n">ssh_cmd</span> <span class="o">+=</span> <span class="n">ssh_tunnel_cmd</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Creating tunnel with cmd: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">ssh_cmd</span><span class="p">)</span>
-
- <span class="n">proc</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">Popen</span><span class="p">(</span><span class="n">ssh_cmd</span><span class="p">,</span>
- <span class="n">stdin</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span>
- <span class="n">stdout</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span>
- <span class="n">close_fds</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
- <span class="n">ready</span> <span class="o">=</span> <span class="n">proc</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
- <span class="k">assert</span> <span class="n">ready</span> <span class="o">==</span> <span class="sa">b</span><span class="s2">"ready"</span><span class="p">,</span> \
- <span class="s2">"Did not get 'ready' from remote, got '</span><span class="si">{0}</span><span class="s2">' instead"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">ready</span><span class="p">)</span>
- <span class="k">yield</span>
- <span class="n">proc</span><span class="o">.</span><span class="n">communicate</span><span class="p">()</span>
- <span class="k">assert</span> <span class="n">proc</span><span class="o">.</span><span class="n">returncode</span> <span class="o">==</span> <span class="mi">0</span><span class="p">,</span> \
- <span class="s2">"Tunnel process did unclean exit (returncode </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">proc</span><span class="o">.</span><span class="n">returncode</span><span class="p">)</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'Creating SSH client for conn_id: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span><span class="p">)</span>
+ <span class="n">client</span> <span class="o">=</span> <span class="n">paramiko</span><span class="o">.</span><span class="n">SSHClient</span><span class="p">()</span>
+ <span class="n">client</span><span class="o">.</span><span class="n">load_system_host_keys</span><span class="p">()</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">no_host_key_check</span><span class="p">:</span>
+ <span class="c1"># Default is RejectPolicy</span>
+ <span class="n">client</span><span class="o">.</span><span class="n">set_missing_host_key_policy</span><span class="p">(</span><span class="n">paramiko</span><span class="o">.</span><span class="n">AutoAddPolicy</span><span class="p">())</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span><span class="o">.</span><span class="n">strip</span><span class="p">():</span>
+ <span class="n">client</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">hostname</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span>
+ <span class="n">username</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">,</span>
+ <span class="n">password</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">password</span><span class="p">,</span>
+ <span class="n">key_filename</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">key_file</span><span class="p">,</span>
+ <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">,</span>
+ <span class="n">compress</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">compress</span><span class="p">,</span>
+ <span class="n">port</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">port</span><span class="p">,</span>
+ <span class="n">sock</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">host_proxy</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">client</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">hostname</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span>
+ <span class="n">username</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">,</span>
+ <span class="n">key_filename</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">key_file</span><span class="p">,</span>
+ <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">,</span>
+ <span class="n">compress</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">compress</span><span class="p">,</span>
+ <span class="n">port</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">port</span><span class="p">,</span>
+ <span class="n">sock</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">host_proxy</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">keepalive_interval</span><span class="p">:</span>
+ <span class="n">client</span><span class="o">.</span><span class="n">get_transport</span><span class="p">()</span><span class="o">.</span><span class="n">set_keepalive</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">keepalive_interval</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="n">client</span>
+ <span class="k">return</span> <span class="n">client</span></div>
<span class="k">def</span> <span class="nf">__enter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s1">'The contextmanager of SSHHook is deprecated.'</span>
+ <span class="s1">'Please use get_conn() as a contextmanager instead.'</span>
+ <span class="s1">'This method will be removed in Airflow 2.0'</span><span class="p">,</span>
+ <span class="n">category</span><span class="o">=</span><span class="ne">DeprecationWarning</span><span class="p">)</span>
<span class="k">return</span> <span class="bp">self</span>
<span class="k">def</span> <span class="nf">__exit__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">exc_type</span><span class="p">,</span> <span class="n">exc_val</span><span class="p">,</span> <span class="n">exc_tb</span><span class="p">):</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">close</span><span class="p">()</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="kc">None</span>
+
+<div class="viewcode-block" id="SSHHook.get_tunnel"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ssh_hook.SSHHook.get_tunnel">[docs]</a> <span class="k">def</span> <span class="nf">get_tunnel</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">remote_port</span><span class="p">,</span> <span class="n">remote_host</span><span class="o">=</span><span class="s2">"localhost"</span><span class="p">,</spa [...]
+ <span class="sd">"""</span>
+<span class="sd"> Creates a tunnel between two hosts. Like ssh -L <LOCAL_PORT>:host:<REMOTE_PORT>.</span>
+
+<span class="sd"> :param remote_port: The remote port to create a tunnel to</span>
+<span class="sd"> :type remote_port: int</span>
+<span class="sd"> :param remote_host: The remote host to create a tunnel to (default localhost)</span>
+<span class="sd"> :type remote_host: str</span>
+<span class="sd"> :param local_port: The local port to attach the tunnel to</span>
+<span class="sd"> :type local_port: int</span>
+
+<span class="sd"> :return: sshtunnel.SSHTunnelForwarder object</span>
+<span class="sd"> """</span>
+
+ <span class="k">if</span> <span class="n">local_port</span><span class="p">:</span>
+ <span class="n">local_bind_address</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'localhost'</span><span class="p">,</span> <span class="n">local_port</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">local_bind_address</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'localhost'</span><span class="p">,)</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span><span class="o">.</span><span class="n">strip</span><span class="p">():</span>
+ <span class="n">client</span> <span class="o">=</span> <span class="n">SSHTunnelForwarder</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span>
+ <span class="n">ssh_port</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">port</span><span class="p">,</span>
+ <span class="n">ssh_username</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">,</span>
+ <span class="n">ssh_password</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">password</span><span class="p">,</span>
+ <span class="n">ssh_pkey</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">key_file</span><span class="p">,</span>
+ <span class="n">ssh_proxy</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">host_proxy</span><span class="p">,</span>
+ <span class="n">local_bind_address</span><span class="o">=</span><span class="n">local_bind_address</span><span class="p">,</span>
+ <span class="n">remote_bind_address</span><span class="o">=</span><span class="p">(</span><span class="n">remote_host</span><span class="p">,</span> <span class="n">remote_port</span><span class="p">),</span>
+ <span class="n">logger</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">client</span> <span class="o">=</span> <span class="n">SSHTunnelForwarder</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span>
+ <span class="n">ssh_port</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">port</span><span class="p">,</span>
+ <span class="n">ssh_username</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">,</span>
+ <span class="n">ssh_pkey</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">key_file</span><span class="p">,</span>
+ <span class="n">ssh_proxy</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">host_proxy</span><span class="p">,</span>
+ <span class="n">local_bind_address</span><span class="o">=</span><span class="n">local_bind_address</span><span class="p">,</span>
+ <span class="n">remote_bind_address</span><span class="o">=</span><span class="p">(</span><span class="n">remote_host</span><span class="p">,</span> <span class="n">remote_port</span><span class="p">),</span>
+ <span class="n">host_pkey_directories</span><span class="o">=</span><span class="p">[],</span>
+ <span class="n">logger</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">client</span></div>
+
+ <span class="k">def</span> <span class="nf">create_tunnel</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">local_port</span><span class="p">,</span> <span class="n">remote_port</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">remote_host</span><span class="o">=</span><span class="s2">"localhost"</span><span class="p">):</span>
+ <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s1">'SSHHook.create_tunnel is deprecated, Please'</span>
+ <span class="s1">'use get_tunnel() instead. But please note that the'</span>
+ <span class="s1">'order of the parameters have changed'</span>
+ <span class="s1">'This method will be removed in Airflow 2.0'</span><span class="p">,</span>
+ <span class="n">category</span><span class="o">=</span><span class="ne">DeprecationWarning</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_tunnel</span><span class="p">(</span><span class="n">remote_port</span><span class="p">,</span> <span class="n">remote_host</span><span class="p">,</span> <span class="n">local_port</span><span class="p">)</span></div>
</pre></div>
</div>
@@ -422,20 +428,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/vertica_hook.html b/_modules/airflow/contrib/hooks/vertica_hook.html
index f60fa92..8193e44 100644
--- a/_modules/airflow/contrib/hooks/vertica_hook.html
+++ b/_modules/airflow/contrib/hooks/vertica_hook.html
@@ -242,20 +242,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/wasb_hook.html b/_modules/airflow/contrib/hooks/wasb_hook.html
index e98b5cf..13007c9 100644
--- a/_modules/airflow/contrib/hooks/wasb_hook.html
+++ b/_modules/airflow/contrib/hooks/wasb_hook.html
@@ -179,6 +179,7 @@
<span class="c1"># under the License.</span>
<span class="c1">#</span>
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">AirflowException</span>
<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
<span class="kn">from</span> <span class="nn">azure.storage.blob</span> <span class="k">import</span> <span class="n">BlockBlobService</span>
@@ -308,7 +309,47 @@
<span class="sd"> """</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">get_blob_to_text</span><span class="p">(</span><span class="n">container_name</span><span class="p">,</span>
<span class="n">blob_name</span><span class="p">,</span>
- <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span><span class="o">.</span><span class="n">content</span></div></div>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span><span class="o">.</span><span class="n">content</span></div>
+
+<div class="viewcode-block" id="WasbHook.delete_file"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.wasb_hook.WasbHook.delete_file">[docs]</a> <span class="k">def</span> <span class="nf">delete_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">container_name</span><span class="p">,</span> <span class="n">blob_name</span><span class="p">,</span> <span class="n">is_prefix</span><span class="o">=</s [...]
+ <span class="n">ignore_if_missing</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Delete a file from Azure Blob Storage.</span>
+
+<span class="sd"> :param container_name: Name of the container.</span>
+<span class="sd"> :type container_name: str</span>
+<span class="sd"> :param blob_name: Name of the blob.</span>
+<span class="sd"> :type blob_name: str</span>
+<span class="sd"> :param is_prefix: If blob_name is a prefix, delete all matching files</span>
+<span class="sd"> :type is_prefix: bool</span>
+<span class="sd"> :param ignore_if_missing: if True, then return success even if the</span>
+<span class="sd"> blob does not exist.</span>
+<span class="sd"> :type ignore_if_missing: bool</span>
+<span class="sd"> :param kwargs: Optional keyword arguments that</span>
+<span class="sd"> `BlockBlobService.create_blob_from_path()` takes.</span>
+<span class="sd"> :type kwargs: object</span>
+<span class="sd"> """</span>
+
+ <span class="k">if</span> <span class="n">is_prefix</span><span class="p">:</span>
+ <span class="n">blobs_to_delete</span> <span class="o">=</span> <span class="p">[</span>
+ <span class="n">blob</span><span class="o">.</span><span class="n">name</span> <span class="k">for</span> <span class="n">blob</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">list_blobs</span><span class="p">(</span>
+ <span class="n">container_name</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="n">blob_name</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span>
+ <span class="p">)</span>
+ <span class="p">]</span>
+ <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">check_for_blob</span><span class="p">(</span><span class="n">container_name</span><span class="p">,</span> <span class="n">blob_name</span><span class="p">):</span>
+ <span class="n">blobs_to_delete</span> <span class="o">=</span> <span class="p">[</span><span class="n">blob_name</span><span class="p">]</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">blobs_to_delete</span> <span class="o">=</span> <span class="p">[]</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">ignore_if_missing</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">blobs_to_delete</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Blob(s) not found: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">blob_name</span><span class="p">))</span>
+
+ <span class="k">for</span> <span class="n">blob_uri</span> <span class="ow">in</span> <span class="n">blobs_to_delete</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Deleting blob: "</span> <span class="o">+</span> <span class="n">blob_uri</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">delete_blob</span><span class="p">(</span><span class="n">container_name</span><span class="p">,</span>
+ <span class="n">blob_uri</span><span class="p">,</span>
+ <span class="n">delete_snapshots</span><span class="o">=</span><span class="s1">'include'</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div></div>
</pre></div>
</div>
@@ -339,20 +380,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/hooks/winrm_hook.html b/_modules/airflow/contrib/hooks/winrm_hook.html
index c89917c..5496cb2 100644
--- a/_modules/airflow/contrib/hooks/winrm_hook.html
+++ b/_modules/airflow/contrib/hooks/winrm_hook.html
@@ -326,20 +326,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/kubernetes/secret.html b/_modules/airflow/contrib/kubernetes/secret.html
index 4d77038..6f75ac3 100644
--- a/_modules/airflow/contrib/kubernetes/secret.html
+++ b/_modules/airflow/contrib/kubernetes/secret.html
@@ -230,20 +230,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/operators/awsbatch_operator.html b/_modules/airflow/contrib/operators/awsbatch_operator.html
index 9f65bb3..2917d61 100644
--- a/_modules/airflow/contrib/operators/awsbatch_operator.html
+++ b/_modules/airflow/contrib/operators/awsbatch_operator.html
@@ -203,18 +203,20 @@
<span class="sd"> :type job_definition: str</span>
<span class="sd"> :param job_queue: the queue name on AWS Batch</span>
<span class="sd"> :type job_queue: str</span>
-<span class="sd"> :param: overrides: the same parameter that boto3 will receive on</span>
-<span class="sd"> containerOverrides (templated):</span>
-<span class="sd"> http://boto3.readthedocs.io/en/latest/reference/services/batch.html#submit_job</span>
-<span class="sd"> :type: overrides: dict</span>
-<span class="sd"> :param max_retries: exponential backoff retries while waiter is not merged, 4200 = 48 hours</span>
+<span class="sd"> :param overrides: the same parameter that boto3 will receive on</span>
+<span class="sd"> containerOverrides (templated).</span>
+<span class="sd"> http://boto3.readthedocs.io/en/latest/reference/services/batch.html#submit_job</span>
+<span class="sd"> :type overrides: dict</span>
+<span class="sd"> :param max_retries: exponential backoff retries while waiter is not merged,</span>
+<span class="sd"> 4200 = 48 hours</span>
<span class="sd"> :type max_retries: int</span>
<span class="sd"> :param aws_conn_id: connection id of AWS credentials / region name. If None,</span>
-<span class="sd"> credential boto3 strategy will be used</span>
-<span class="sd"> (http://boto3.readthedocs.io/en/latest/guide/configuration.html).</span>
+<span class="sd"> credential boto3 strategy will be used</span>
+<span class="sd"> (http://boto3.readthedocs.io/en/latest/guide/configuration.html).</span>
<span class="sd"> :type aws_conn_id: str</span>
<span class="sd"> :param region_name: region name to use in AWS Hook.</span>
<span class="sd"> Override the region_name in connection (if provided)</span>
+<span class="sd"> :type region_name: str</span>
<span class="sd"> """</span>
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#c3dae0'</span>
@@ -240,7 +242,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="AWSBatchOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.awsbatch_operator.AWSBatchOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
<span class="s1">'Running AWS Batch Job - Job definition: </span><span class="si">%s</span><span class="s1"> - on queue </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">job_definition</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_queue</span>
@@ -271,7 +273,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'AWS Batch Job has been successfully executed: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">response</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'AWS Batch Job has failed executed'</span><span class="p">)</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="n">e</span><span class="p">)</span></div>
<span class="k">def</span> <span class="nf">_wait_for_task_ended</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -300,7 +302,7 @@
<span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s1">'jobs'</span><span class="p">][</span><span class="o">-</span><span class="mi">1</span><span class="p">][</span><span class="s1">'status'</span><span class="p">]</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'SUCCEEDED'</span><span class="p">,</span> <span class="s1">'FAILED'</span><span class="p">]:</span>
<span class="n">retry</span> <span class="o">=</span> <span class="kc">False</span>
- <span class="n">sleep</span><span class="p">(</span> <span class="mi">1</span> <span class="o">+</span> <span class="nb">pow</span><span class="p">(</span><span class="n">retries</span> <span class="o">*</span> <span class="mf">0.1</span><span class="p">,</span> <span class="mi">2</span><span class="p">))</span>
+ <span class="n">sleep</span><span class="p">(</span><span class="mi">1</span> <span class="o">+</span> <span class="nb">pow</span><span class="p">(</span><span class="n">retries</span> <span class="o">*</span> <span class="mf">0.1</span><span class="p">,</span> <span class="mi">2</span><span class="p">))</span>
<span class="n">retries</span> <span class="o">+=</span> <span class="mi">1</span>
<span class="k">def</span> <span class="nf">_check_success_task</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
@@ -330,12 +332,12 @@
<span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span>
<span class="p">)</span>
- <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<div class="viewcode-block" id="AWSBatchOperator.on_kill"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.awsbatch_operator.AWSBatchOperator.on_kill">[docs]</a> <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">terminate_job</span><span class="p">(</span>
<span class="n">jobId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">jobId</span><span class="p">,</span>
<span class="n">reason</span><span class="o">=</span><span class="s1">'Task killed by the user'</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">response</span><span class="p">)</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">response</span><span class="p">)</span></div></div>
</pre></div>
</div>
@@ -366,20 +368,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/operators/bigquery_check_operator.html b/_modules/airflow/contrib/operators/bigquery_check_operator.html
index 9202dec..217dbeb 100644
--- a/_modules/airflow/contrib/operators/bigquery_check_operator.html
+++ b/_modules/airflow/contrib/operators/bigquery_check_operator.html
@@ -216,21 +216,25 @@
<span class="sd"> :type sql: string</span>
<span class="sd"> :param bigquery_conn_id: reference to the BigQuery database</span>
<span class="sd"> :type bigquery_conn_id: string</span>
+<span class="sd"> :param use_legacy_sql: Whether to use legacy SQL (true)</span>
+<span class="sd"> or standard SQL (false).</span>
+<span class="sd"> :type use_legacy_sql: boolean</span>
<span class="sd"> """</span>
<span class="nd">@apply_defaults</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
- <span class="bp">self</span><span class="p">,</span>
- <span class="n">sql</span><span class="p">,</span>
- <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">'bigquery_default'</span><span class="p">,</span>
- <span class="o">*</span><span class="n">args</span><span class="p">,</span>
- <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">sql</span><span class="p">,</span>
+ <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">'bigquery_default'</span><span class="p">,</span>
+ <span class="n">use_legacy_sql</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">BigQueryCheckOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">sql</span><span class="o">=</span><span class="n">sql</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span [...]
<span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span> <span class="o">=</span> <span class="n">use_legacy_sql</span>
<span class="k">def</span> <span class="nf">get_db_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="k">return</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">)</span></div>
+ <span class="k">return</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
+ <span class="n">use_legacy_sql</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span><span class="p">)</span></div>
<div class="viewcode-block" id="BigQueryValueCheckOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_check_operator.BigQueryValueCheckOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryValueCheckOperator</span><span class="p">(</span><span class="n">ValueCheckOperator</span><span class="p">):</span>
@@ -239,20 +243,27 @@
<span class="sd"> :param sql: the sql to be executed</span>
<span class="sd"> :type sql: string</span>
+<span class="sd"> :param use_legacy_sql: Whether to use legacy SQL (true)</span>
+<span class="sd"> or standard SQL (false).</span>
+<span class="sd"> :type use_legacy_sql: boolean</span>
<span class="sd"> """</span>
<span class="nd">@apply_defaults</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
- <span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">pass_value</span><span class="p">,</span> <span class="n">tolerance</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">'bigquery_default'</span><span class="p">,</span>
- <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span>
+ <span class="n">pass_value</span><span class="p">,</span>
+ <span class="n">tolerance</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">'bigquery_default'</span><span class="p">,</span>
+ <span class="n">use_legacy_sql</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">BigQueryValueCheckOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
<span class="n">sql</span><span class="o">=</span><span class="n">sql</span><span class="p">,</span> <span class="n">pass_value</span><span class="o">=</span><span class="n">pass_value</span><span class="p">,</span> <span class="n">tolerance</span><span class="o">=</span><span class="n">tolerance</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span> <span class="o">=</span> <span class="n">use_legacy_sql</span>
<span class="k">def</span> <span class="nf">get_db_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="k">return</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">)</span></div>
+ <span class="k">return</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
+ <span class="n">use_legacy_sql</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span><span class="p">)</span></div>
<div class="viewcode-block" id="BigQueryIntervalCheckOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_check_operator.BigQueryIntervalCheckOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryIntervalCheckOperator</span><span class="p">(</span><span class="n">IntervalCheckOperator</span><span class="p">):</span>
@@ -274,22 +285,25 @@
<span class="sd"> example 'COUNT(*)': 1.5 would require a 50 percent or less difference</span>
<span class="sd"> between the current day, and the prior days_back.</span>
<span class="sd"> :type metrics_threshold: dict</span>
+<span class="sd"> :param use_legacy_sql: Whether to use legacy SQL (true)</span>
+<span class="sd"> or standard SQL (false).</span>
+<span class="sd"> :type use_legacy_sql: boolean</span>
<span class="sd"> """</span>
<span class="nd">@apply_defaults</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
- <span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">metrics_thresholds</span><span class="p">,</span>
- <span class="n">date_filter_column</span><span class="o">=</span><span class="s1">'ds'</span><span class="p">,</span> <span class="n">days_back</span><span class="o">=-</span><span class="mi">7</span><span class="p">,</span>
- <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">'bigquery_default'</span><span class="p">,</span>
- <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">metrics_thresholds</span><span class="p">,</span> <span class="n">date_filter_column</span><span class="o">=</span><span class="s1">'ds'</span><span class="p">,</span>
+ <span class="n">days_back</span><span class="o">=-</span><span class="mi">7</span><span class="p">,</span> <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">'bigquery_default'</span><span class="p">,</span>
+ <span class="n">use_legacy_sql</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">BigQueryIntervalCheckOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
<span class="n">table</span><span class="o">=</span><span class="n">table</span><span class="p">,</span> <span class="n">metrics_thresholds</span><span class="o">=</span><span class="n">metrics_thresholds</span><span class="p">,</span>
<span class="n">date_filter_column</span><span class="o">=</span><span class="n">date_filter_column</span><span class="p">,</span> <span class="n">days_back</span><span class="o">=</span><span class="n">days_back</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span> <span class="o">=</span> <span class="n">use_legacy_sql</span>
<span class="k">def</span> <span class="nf">get_db_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="k">return</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">)</span></div>
+ <span class="k">return</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
+ <span class="n">use_legacy_sql</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span><span class="p">)</span></div>
</pre></div>
</div>
@@ -320,20 +334,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/operators/bigquery_get_data.html b/_modules/airflow/contrib/operators/bigquery_get_data.html
index abff695..7e3da45 100644
--- a/_modules/airflow/contrib/operators/bigquery_get_data.html
+++ b/_modules/airflow/contrib/operators/bigquery_get_data.html
@@ -249,7 +249,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="BigQueryGetDataOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_get_data.BigQueryGetDataOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Fetching Data from:'</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Dataset: </span><span class="si">%s</span><span class="s1"> ; Table: </span><span class="si">%s</span><span class="s1"> ; Max Results: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dataset_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_results</span><span class="p">)</span>
@@ -274,7 +274,7 @@
<span class="n">single_row</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">fields</span><span class="p">[</span><span class="s1">'v'</span><span class="p">])</span>
<span class="n">table_data</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">single_row</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">table_data</span></div>
+ <span class="k">return</span> <span class="n">table_data</span></div></div>
</pre></div>
</div>
@@ -305,20 +305,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/operators/bigquery_operator.html b/_modules/airflow/contrib/operators/bigquery_operator.html
index f4a00d9..3ee4a10 100644
--- a/_modules/airflow/contrib/operators/bigquery_operator.html
+++ b/_modules/airflow/contrib/operators/bigquery_operator.html
@@ -242,18 +242,19 @@
<span class="sd"> :param query_params: a dictionary containing query parameter types and</span>
<span class="sd"> values, passed to BigQuery.</span>
<span class="sd"> :type query_params: dict</span>
+<span class="sd"> :param labels: a dictionary containing labels for the job/query,</span>
+<span class="sd"> passed to BigQuery</span>
+<span class="sd"> :type labels: dict</span>
<span class="sd"> :param priority: Specifies a priority for the query.</span>
<span class="sd"> Possible values include INTERACTIVE and BATCH.</span>
<span class="sd"> The default value is INTERACTIVE.</span>
<span class="sd"> :type priority: string</span>
<span class="sd"> :param time_partitioning: configure optional time partitioning fields i.e.</span>
-<span class="sd"> partition by field, type and</span>
-<span class="sd"> expiration as per API specifications. Note that 'field' is not available in</span>
-<span class="sd"> conjunction with dataset.table$partition.</span>
+<span class="sd"> partition by field, type and expiration as per API specifications.</span>
<span class="sd"> :type time_partitioning: dict</span>
<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bql'</span><span class="p">,</span> <span class="s1">'sql'</span><span class="p">,</span> <span class="s1">'destination_dataset_table'</span><span class="p">)</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bql'</span><span class="p">,</span> <span class="s1">'sql'</span><span class="p">,</span> <span class="s1">'destination_dataset_table'</span><span class="p">,</span> <span class="s1">'labels'</span><span class="p">)</span>
<span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,</span> <span class="p">)</span>
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#e4f0e8'</span>
@@ -264,7 +265,7 @@
<span class="n">destination_dataset_table</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">write_disposition</span><span class="o">=</span><span class="s1">'WRITE_EMPTY'</span><span class="p">,</span>
<span class="n">allow_large_results</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">flatten_results</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">flatten_results</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">'bigquery_default'</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">udf_config</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
@@ -274,6 +275,7 @@
<span class="n">create_disposition</span><span class="o">=</span><span class="s1">'CREATE_IF_NEEDED'</span><span class="p">,</span>
<span class="n">schema_update_options</span><span class="o">=</span><span class="p">(),</span>
<span class="n">query_params</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">priority</span><span class="o">=</span><span class="s1">'INTERACTIVE'</span><span class="p">,</span>
<span class="n">time_partitioning</span><span class="o">=</span><span class="p">{},</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span>
@@ -294,6 +296,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">maximum_bytes_billed</span> <span class="o">=</span> <span class="n">maximum_bytes_billed</span>
<span class="bp">self</span><span class="o">.</span><span class="n">schema_update_options</span> <span class="o">=</span> <span class="n">schema_update_options</span>
<span class="bp">self</span><span class="o">.</span><span class="n">query_params</span> <span class="o">=</span> <span class="n">query_params</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="o">=</span> <span class="n">labels</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bq_cursor</span> <span class="o">=</span> <span class="kc">None</span>
<span class="bp">self</span><span class="o">.</span><span class="n">priority</span> <span class="o">=</span> <span class="n">priority</span>
<span class="bp">self</span><span class="o">.</span><span class="n">time_partitioning</span> <span class="o">=</span> <span class="n">time_partitioning</span>
@@ -312,7 +315,7 @@
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">'</span><span class="si">{}</span><span class="s1"> missing 1 required positional '</span>
<span class="s1">'argument: `sql`'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">))</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="BigQueryOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_operator.BigQueryOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">bq_cursor</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span>
@@ -332,16 +335,17 @@
<span class="n">maximum_bytes_billed</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">maximum_bytes_billed</span><span class="p">,</span>
<span class="n">create_disposition</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">create_disposition</span><span class="p">,</span>
<span class="n">query_params</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">query_params</span><span class="p">,</span>
+ <span class="n">labels</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">labels</span><span class="p">,</span>
<span class="n">schema_update_options</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">schema_update_options</span><span class="p">,</span>
<span class="n">priority</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">priority</span><span class="p">,</span>
<span class="n">time_partitioning</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">time_partitioning</span>
- <span class="p">)</span>
+ <span class="p">)</span></div>
- <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<div class="viewcode-block" id="BigQueryOperator.on_kill"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_operator.BigQueryOperator.on_kill">[docs]</a> <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">BigQueryOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">on_kill</span><span class="p">()</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">bq_cursor</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Canceling running query due to execution timeout'</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">bq_cursor</span><span class="o">.</span><span class="n">cancel_query</span><span class="p">()</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bq_cursor</span><span class="o">.</span><span class="n">cancel_query</span><span class="p">()</span></div></div>
<div class="viewcode-block" id="BigQueryCreateEmptyTableOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_operator.BigQueryCreateEmptyTableOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryCreateEmptyTableOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
@@ -389,49 +393,52 @@
<span class="sd"> work, the service account making the request must have domain-wide</span>
<span class="sd"> delegation enabled.</span>
<span class="sd"> :type delegate_to: string</span>
-
-<span class="sd"> **Example (with schema JSON in GCS)**: ::</span>
-
-<span class="sd"> CreateTable = BigQueryCreateEmptyTableOperator(</span>
-<span class="sd"> task_id='BigQueryCreateEmptyTableOperator_task',</span>
-<span class="sd"> dataset_id='ODS',</span>
-<span class="sd"> table_id='Employees',</span>
-<span class="sd"> project_id='internal-gcp-project',</span>
-<span class="sd"> gcs_schema_object='gs://schema-bucket/employee_schema.json',</span>
-<span class="sd"> bigquery_conn_id='airflow-service-account',</span>
-<span class="sd"> google_cloud_storage_conn_id='airflow-service-account'</span>
-<span class="sd"> )</span>
-
-<span class="sd"> **Corresponding Schema file** (``employee_schema.json``): ::</span>
-
-<span class="sd"> [</span>
-<span class="sd"> {</span>
-<span class="sd"> "mode": "NULLABLE",</span>
-<span class="sd"> "name": "emp_name",</span>
-<span class="sd"> "type": "STRING"</span>
-<span class="sd"> },</span>
-<span class="sd"> {</span>
-<span class="sd"> "mode": "REQUIRED",</span>
-<span class="sd"> "name": "salary",</span>
-<span class="sd"> "type": "INTEGER"</span>
-<span class="sd"> }</span>
-<span class="sd"> ]</span>
-
-<span class="sd"> **Example (with schema in the DAG)**: ::</span>
-
-<span class="sd"> CreateTable = BigQueryCreateEmptyTableOperator(</span>
-<span class="sd"> task_id='BigQueryCreateEmptyTableOperator_task',</span>
-<span class="sd"> dataset_id='ODS',</span>
-<span class="sd"> table_id='Employees',</span>
-<span class="sd"> project_id='internal-gcp-project',</span>
-<span class="sd"> schema_fields=[{"name": "emp_name", "type": "STRING", "mode": "REQUIRED"},</span>
-<span class="sd"> {"name": "salary", "type": "INTEGER", "mode": "NULLABLE"}],</span>
-<span class="sd"> bigquery_conn_id='airflow-service-account',</span>
-<span class="sd"> google_cloud_storage_conn_id='airflow-service-account'</span>
-<span class="sd"> )</span>
+<span class="sd"> :param labels: a dictionary containing labels for the table, passed to BigQuery</span>
+
+<span class="sd"> **Example (with schema JSON in GCS)**: ::</span>
+
+<span class="sd"> CreateTable = BigQueryCreateEmptyTableOperator(</span>
+<span class="sd"> task_id='BigQueryCreateEmptyTableOperator_task',</span>
+<span class="sd"> dataset_id='ODS',</span>
+<span class="sd"> table_id='Employees',</span>
+<span class="sd"> project_id='internal-gcp-project',</span>
+<span class="sd"> gcs_schema_object='gs://schema-bucket/employee_schema.json',</span>
+<span class="sd"> bigquery_conn_id='airflow-service-account',</span>
+<span class="sd"> google_cloud_storage_conn_id='airflow-service-account'</span>
+<span class="sd"> )</span>
+
+<span class="sd"> **Corresponding Schema file** (``employee_schema.json``): ::</span>
+
+<span class="sd"> [</span>
+<span class="sd"> {</span>
+<span class="sd"> "mode": "NULLABLE",</span>
+<span class="sd"> "name": "emp_name",</span>
+<span class="sd"> "type": "STRING"</span>
+<span class="sd"> },</span>
+<span class="sd"> {</span>
+<span class="sd"> "mode": "REQUIRED",</span>
+<span class="sd"> "name": "salary",</span>
+<span class="sd"> "type": "INTEGER"</span>
+<span class="sd"> }</span>
+<span class="sd"> ]</span>
+
+<span class="sd"> **Example (with schema in the DAG)**: ::</span>
+
+<span class="sd"> CreateTable = BigQueryCreateEmptyTableOperator(</span>
+<span class="sd"> task_id='BigQueryCreateEmptyTableOperator_task',</span>
+<span class="sd"> dataset_id='ODS',</span>
+<span class="sd"> table_id='Employees',</span>
+<span class="sd"> project_id='internal-gcp-project',</span>
+<span class="sd"> schema_fields=[{"name": "emp_name", "type": "STRING", "mode": "REQUIRED"},</span>
+<span class="sd"> {"name": "salary", "type": "INTEGER", "mode": "NULLABLE"}],</span>
+<span class="sd"> bigquery_conn_id='airflow-service-account',</span>
+<span class="sd"> google_cloud_storage_conn_id='airflow-service-account'</span>
+<span class="sd"> )</span>
+<span class="sd"> :type labels: dict</span>
<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'dataset_id'</span><span class="p">,</span> <span class="s1">'table_id'</span><span class="p">,</span> <span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'gcs_schema_object'</span><span class="p">)</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'dataset_id'</span><span class="p">,</span> <span class="s1">'table_id'</span><span class="p">,</span> <span class="s1">'project_id'</span><span class="p">,</span>
+ <span class="s1">'gcs_schema_object'</span><span class="p">,</span> <span class="s1">'labels'</span><span class="p">)</span>
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f0eee4'</span>
<span class="nd">@apply_defaults</span>
@@ -445,6 +452,7 @@
<span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">'bigquery_default'</span><span class="p">,</span>
<span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">BigQueryCreateEmptyTableOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
@@ -458,8 +466,9 @@
<span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span> <span class="o">=</span> <span class="n">google_cloud_storage_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
<span class="bp">self</span><span class="o">.</span><span class="n">time_partitioning</span> <span class="o">=</span> <span class="n">time_partitioning</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="o">=</span> <span class="n">labels</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="BigQueryCreateEmptyTableOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_operator.BigQueryCreateEmptyTableOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">bq_hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
@@ -484,8 +493,9 @@
<span class="n">dataset_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_id</span><span class="p">,</span>
<span class="n">table_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_id</span><span class="p">,</span>
<span class="n">schema_fields</span><span class="o">=</span><span class="n">schema_fields</span><span class="p">,</span>
- <span class="n">time_partitioning</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">time_partitioning</span>
- <span class="p">)</span></div>
+ <span class="n">time_partitioning</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">time_partitioning</span><span class="p">,</span>
+ <span class="n">labels</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">labels</span>
+ <span class="p">)</span></div></div>
<div class="viewcode-block" id="BigQueryCreateExternalTableOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_operator.BigQueryCreateExternalTableOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryCreateExternalTableOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
@@ -557,9 +567,11 @@
<span class="sd"> :type delegate_to: string</span>
<span class="sd"> :param src_fmt_configs: configure optional fields specific to the source format</span>
<span class="sd"> :type src_fmt_configs: dict</span>
+<span class="sd"> :param labels a dictionary containing labels for the table, passed to BigQuery</span>
+<span class="sd"> :type labels: dict</span>
<span class="sd"> """</span>
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bucket'</span><span class="p">,</span> <span class="s1">'source_objects'</span><span class="p">,</span>
- <span class="s1">'schema_object'</span><span class="p">,</span> <span class="s1">'destination_project_dataset_table'</span><span class="p">)</span>
+ <span class="s1">'schema_object'</span><span class="p">,</span> <span class="s1">'destination_project_dataset_table'</span><span class="p">,</span> <span class="s1">'labels'</span><span class="p">)</span>
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f0eee4'</span>
<span class="nd">@apply_defaults</span>
@@ -581,6 +593,7 @@
<span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">src_fmt_configs</span><span class="o">=</span><span class="p">{},</span>
+ <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">BigQueryCreateExternalTableOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
@@ -607,8 +620,9 @@
<span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
<span class="bp">self</span><span class="o">.</span><span class="n">src_fmt_configs</span> <span class="o">=</span> <span class="n">src_fmt_configs</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="o">=</span> <span class="n">labels</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="BigQueryCreateExternalTableOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_operator.BigQueryCreateExternalTableOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">bq_hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
@@ -640,8 +654,61 @@
<span class="n">quote_character</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">quote_character</span><span class="p">,</span>
<span class="n">allow_quoted_newlines</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">allow_quoted_newlines</span><span class="p">,</span>
<span class="n">allow_jagged_rows</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">allow_jagged_rows</span><span class="p">,</span>
- <span class="n">src_fmt_configs</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">src_fmt_configs</span>
- <span class="p">)</span></div>
+ <span class="n">src_fmt_configs</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">src_fmt_configs</span><span class="p">,</span>
+ <span class="n">labels</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">labels</span>
+ <span class="p">)</span></div></div>
+
+
+<div class="viewcode-block" id="BigQueryDeleteDatasetOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_operator.BigQueryDeleteDatasetOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryDeleteDatasetOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">""""</span>
+<span class="sd"> This operator deletes an existing dataset from your Project in Big query.</span>
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets/delete</span>
+<span class="sd"> :param project_id: The project id of the dataset.</span>
+<span class="sd"> :type project_id: string</span>
+<span class="sd"> :param dataset_id: The dataset to be deleted.</span>
+<span class="sd"> :type dataset_id: string</span>
+
+<span class="sd"> **Example**: ::</span>
+
+<span class="sd"> delete_temp_data = BigQueryDeleteDatasetOperator(</span>
+<span class="sd"> dataset_id = 'temp-dataset',</span>
+<span class="sd"> project_id = 'temp-project',</span>
+<span class="sd"> bigquery_conn_id='_my_gcp_conn_',</span>
+<span class="sd"> task_id='Deletetemp',</span>
+<span class="sd"> dag=dag)</span>
+<span class="sd"> """</span>
+
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'dataset_id'</span><span class="p">,</span> <span class="s1">'project_id'</span><span class="p">)</span>
+ <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f00004'</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">dataset_id</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">'bigquery_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">dataset_id</span> <span class="o">=</span> <span class="n">dataset_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Dataset id: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Project id: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">)</span>
+
+ <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryDeleteDatasetOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+<div class="viewcode-block" id="BigQueryDeleteDatasetOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_operator.BigQueryDeleteDatasetOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="n">bq_hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+
+ <span class="n">conn</span> <span class="o">=</span> <span class="n">bq_hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="n">cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
+
+ <span class="n">cursor</span><span class="o">.</span><span class="n">delete_dataset</span><span class="p">(</span>
+ <span class="n">project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">dataset_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_id</span>
+ <span class="p">)</span></div></div>
</pre></div>
</div>
@@ -672,20 +739,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/operators/bigquery_table_delete_operator.html b/_modules/airflow/contrib/operators/bigquery_table_delete_operator.html
index 8b3a424..0211360 100644
--- a/_modules/airflow/contrib/operators/bigquery_table_delete_operator.html
+++ b/_modules/airflow/contrib/operators/bigquery_table_delete_operator.html
@@ -218,13 +218,13 @@
<span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
<span class="bp">self</span><span class="o">.</span><span class="n">ignore_if_missing</span> <span class="o">=</span> <span class="n">ignore_if_missing</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="BigQueryTableDeleteOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_table_delete_operator.BigQueryTableDeleteOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Deleting: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">deletion_dataset_table</span><span class="p">)</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
<span class="n">conn</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
<span class="n">cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
- <span class="n">cursor</span><span class="o">.</span><span class="n">run_table_delete</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">deletion_dataset_table</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ignore_if_missing</span><span class="p">)</span></div>
+ <span class="n">cursor</span><span class="o">.</span><span class="n">run_table_delete</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">deletion_dataset_table</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ignore_if_missing</span><span class="p">)</span></div></div>
</pre></div>
</div>
@@ -255,20 +255,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/operators/bigquery_to_bigquery.html b/_modules/airflow/contrib/operators/bigquery_to_bigquery.html
index da4edcd..fb16a0c 100644
--- a/_modules/airflow/contrib/operators/bigquery_to_bigquery.html
+++ b/_modules/airflow/contrib/operators/bigquery_to_bigquery.html
@@ -210,9 +210,12 @@
<span class="sd"> For this to work, the service account making the request must have domain-wide</span>
<span class="sd"> delegation enabled.</span>
<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> :param labels: a dictionary containing labels for the job/query,</span>
+<span class="sd"> passed to BigQuery</span>
+<span class="sd"> :type labels: dict</span>
<span class="sd"> """</span>
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'source_project_dataset_tables'</span><span class="p">,</span>
- <span class="s1">'destination_project_dataset_table'</span><span class="p">)</span>
+ <span class="s1">'destination_project_dataset_table'</span><span class="p">,</span> <span class="s1">'labels'</span><span class="p">)</span>
<span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,)</span>
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#e6f0e4'</span>
@@ -224,6 +227,7 @@
<span class="n">create_disposition</span><span class="o">=</span><span class="s1">'CREATE_IF_NEEDED'</span><span class="p">,</span>
<span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">'bigquery_default'</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span>
<span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">BigQueryToBigQueryOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
@@ -233,8 +237,9 @@
<span class="bp">self</span><span class="o">.</span><span class="n">create_disposition</span> <span class="o">=</span> <span class="n">create_disposition</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="o">=</span> <span class="n">labels</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="BigQueryToBigQueryOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_to_bigquery.BigQueryToBigQueryOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
<span class="s1">'Executing copy of </span><span class="si">%s</span><span class="s1"> into: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">source_project_dataset_tables</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">destination_project_dataset_table</span>
@@ -247,7 +252,8 @@
<span class="bp">self</span><span class="o">.</span><span class="n">source_project_dataset_tables</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">destination_project_dataset_table</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">write_disposition</span><span class="p">,</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">create_disposition</span><span class="p">)</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="n">create_disposition</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">labels</span><span class="p">)</span></div></div>
</pre></div>
</div>
@@ -278,20 +284,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/operators/bigquery_to_gcs.html b/_modules/airflow/contrib/operators/bigquery_to_gcs.html
index 9052db6..87a2a29 100644
--- a/_modules/airflow/contrib/operators/bigquery_to_gcs.html
+++ b/_modules/airflow/contrib/operators/bigquery_to_gcs.html
@@ -215,8 +215,12 @@
<span class="sd"> For this to work, the service account making the request must have domain-wide</span>
<span class="sd"> delegation enabled.</span>
<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> :param labels: a dictionary containing labels for the job/query,</span>
+<span class="sd"> passed to BigQuery</span>
+<span class="sd"> :type labels: dict</span>
<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'source_project_dataset_table'</span><span class="p">,</span> <span class="s1">'destination_cloud_storage_uris'</span><span class="p">)</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'source_project_dataset_table'</span><span class="p">,</span>
+ <span class="s1">'destination_cloud_storage_uris'</span><span class="p">,</span> <span class="s1">'labels'</span><span class="p">)</span>
<span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,)</span>
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#e4e6f0'</span>
@@ -230,6 +234,7 @@
<span class="n">print_header</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">'bigquery_default'</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span>
<span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">BigQueryToCloudStorageOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
@@ -241,8 +246,9 @@
<span class="bp">self</span><span class="o">.</span><span class="n">print_header</span> <span class="o">=</span> <span class="n">print_header</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="o">=</span> <span class="n">labels</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="BigQueryToCloudStorageOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_to_gcs.BigQueryToCloudStorageOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing extract of </span><span class="si">%s</span><span class="s1"> into: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">source_project_dataset_table</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">destination_cloud_storage_uris</span><span class="p">)</span>
@@ -256,7 +262,8 @@
<span class="bp">self</span><span class="o">.</span><span class="n">compression</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">export_format</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">field_delimiter</span><span class="p">,</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">print_header</span><span class="p">)</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="n">print_header</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">labels</span><span class="p">)</span></div></div>
</pre></div>
</div>
@@ -287,20 +294,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/operators/cassandra_to_gcs.html b/_modules/airflow/contrib/operators/cassandra_to_gcs.html
index a53bc3d..73b5b34 100644
--- a/_modules/airflow/contrib/operators/cassandra_to_gcs.html
+++ b/_modules/airflow/contrib/operators/cassandra_to_gcs.html
@@ -287,7 +287,7 @@
<span class="s1">'VarcharType'</span><span class="p">:</span> <span class="s1">'STRING'</span><span class="p">,</span>
<span class="p">}</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="CassandraToGoogleCloudStorageOperator.execute"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.cassandra_to_gcs.CassandraToGoogleCloudStorageOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">cursor</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_query_cassandra</span><span class="p">()</span>
<span class="n">files_to_upload</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_write_local_data_files</span><span class="p">(</span><span class="n">cursor</span><span class="p">)</span>
@@ -306,7 +306,7 @@
<span class="n">file_handle</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
<span class="c1"># Close all sessions and connection associated with this Cassandra cluster</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">shutdown_cluster</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">shutdown_cluster</span><span class="p">()</span></div>
<span class="k">def</span> <span class="nf">_query_cassandra</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -544,20 +544,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/operators/databricks_operator.html b/_modules/airflow/contrib/operators/databricks_operator.html
index f401636..51c13d7 100644
--- a/_modules/airflow/contrib/operators/databricks_operator.html
+++ b/_modules/airflow/contrib/operators/databricks_operator.html
@@ -307,6 +307,9 @@
<span class="sd"> :param databricks_retry_limit: Amount of times retry if the Databricks backend is</span>
<span class="sd"> unreachable. Its value must be greater than or equal to 1.</span>
<span class="sd"> :type databricks_retry_limit: int</span>
+<span class="sd"> :param databricks_retry_delay: Number of seconds to wait between retries (it</span>
+<span class="sd"> might be a floating point number).</span>
+<span class="sd"> :type databricks_retry_delay: float</span>
<span class="sd"> :param do_xcom_push: Whether we should push run_id and run_page_url to xcom.</span>
<span class="sd"> :type do_xcom_push: boolean</span>
<span class="sd"> """</span>
@@ -329,6 +332,7 @@
<span class="n">databricks_conn_id</span><span class="o">=</span><span class="s1">'databricks_default'</span><span class="p">,</span>
<span class="n">polling_period_seconds</span><span class="o">=</span><span class="mi">30</span><span class="p">,</span>
<span class="n">databricks_retry_limit</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span>
+ <span class="n">databricks_retry_delay</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
<span class="n">do_xcom_push</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -339,6 +343,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn_id</span> <span class="o">=</span> <span class="n">databricks_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">polling_period_seconds</span> <span class="o">=</span> <span class="n">polling_period_seconds</span>
<span class="bp">self</span><span class="o">.</span><span class="n">databricks_retry_limit</span> <span class="o">=</span> <span class="n">databricks_retry_limit</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">databricks_retry_delay</span> <span class="o">=</span> <span class="n">databricks_retry_delay</span>
<span class="k">if</span> <span class="n">spark_jar_task</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">json</span><span class="p">[</span><span class="s1">'spark_jar_task'</span><span class="p">]</span> <span class="o">=</span> <span class="n">spark_jar_task</span>
<span class="k">if</span> <span class="n">notebook_task</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
@@ -393,9 +398,10 @@
<span class="k">def</span> <span class="nf">get_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="n">DatabricksHook</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn_id</span><span class="p">,</span>
- <span class="n">retry_limit</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">databricks_retry_limit</span><span class="p">)</span>
+ <span class="n">retry_limit</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">databricks_retry_limit</span><span class="p">,</span>
+ <span class="n">retry_delay</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">databricks_retry_delay</span><span class="p">)</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="DatabricksSubmitRunOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">hook</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">run_id</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">submit_run</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">json</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">do_xcom_push</span><span class="p">:</span>
@@ -421,15 +427,15 @@
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'</span><span class="si">%s</span><span class="s1"> in run state: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="n">run_state</span><span clas [...]
<span class="bp">self</span><span class="o">.</span><span class="n">_log_run_page_url</span><span class="p">(</span><span class="n">run_page_url</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Sleeping for </span><span class="si">%s</span><span class="s1"> seconds.'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">polling_period_seconds</span><span class="p">)</span>
- <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">polling_period_seconds</span><span class="p">)</span>
+ <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">polling_period_seconds</span><span class="p">)</span></div>
- <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<div class="viewcode-block" id="DatabricksSubmitRunOperator.on_kill"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator.on_kill">[docs]</a> <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">hook</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span>
<span class="n">hook</span><span class="o">.</span><span class="n">cancel_run</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">run_id</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
<span class="s1">'Task: </span><span class="si">%s</span><span class="s1"> with run_id: </span><span class="si">%s</span><span class="s1"> was requested to be cancelled.'</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_id</span>
- <span class="p">)</span></div>
+ <span class="p">)</span></div></div>
</pre></div>
</div>
@@ -460,20 +466,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/operators/dataflow_operator.html b/_modules/airflow/contrib/operators/dataflow_operator.html
index 09cc5c1..59107e2 100644
--- a/_modules/airflow/contrib/operators/dataflow_operator.html
+++ b/_modules/airflow/contrib/operators/dataflow_operator.html
@@ -177,7 +177,7 @@
<span class="c1"># KIND, either express or implied. See the License for the</span>
<span class="c1"># specific language governing permissions and limitations</span>
<span class="c1"># under the License.</span>
-
+<span class="kn">import</span> <span class="nn">os</span>
<span class="kn">import</span> <span class="nn">re</span>
<span class="kn">import</span> <span class="nn">uuid</span>
<span class="kn">import</span> <span class="nn">copy</span>
@@ -291,7 +291,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">poll_sleep</span> <span class="o">=</span> <span class="n">poll_sleep</span>
<span class="bp">self</span><span class="o">.</span><span class="n">job_class</span> <span class="o">=</span> <span class="n">job_class</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="DataFlowJavaOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataflow_operator.DataFlowJavaOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">bucket_helper</span> <span class="o">=</span> <span class="n">GoogleCloudBucketHelper</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">jar</span> <span class="o">=</span> <span class="n">bucket_helper</span><span class="o">.</span><span class="n">google_cloud_to_local</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">jar</span><span class="p">)</span>
@@ -303,7 +303,7 @@
<span class="n">dataflow_options</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">options</span><span class="p">)</span>
<span class="n">hook</span><span class="o">.</span><span class="n">start_java_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="n">dataflow_options</span><span class="p">,</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">jar</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_class</span><span class="p">)</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="n">jar</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_class</span><span class="p">)</span></div></div>
<div class="viewcode-block" id="DataflowTemplateOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataflow_operator.DataflowTemplateOperator">[docs]</a><span class="k">class</span> <span class="nc">DataflowTemplateOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
@@ -403,16 +403,48 @@
<span class="bp">self</span><span class="o">.</span><span class="n">template</span> <span class="o">=</span> <span class="n">template</span>
<span class="bp">self</span><span class="o">.</span><span class="n">parameters</span> <span class="o">=</span> <span class="n">parameters</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="DataflowTemplateOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataflow_operator.DataflowTemplateOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">DataFlowHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">,</span>
<span class="n">poll_sleep</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">poll_sleep</span><span class="p">)</span>
<span class="n">hook</span><span class="o">.</span><span class="n">start_template_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataflow_default_options</span><span class="p">,</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">template</span><span class="p">)</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">template</span><span class="p">)</span></div></div>
<div class="viewcode-block" id="DataFlowPythonOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataflow_operator.DataFlowPythonOperator">[docs]</a><span class="k">class</span> <span class="nc">DataFlowPythonOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Create a new DataFlowPythonOperator. Note that both</span>
+<span class="sd"> dataflow_default_options and options will be merged to specify pipeline</span>
+<span class="sd"> execution parameter, and dataflow_default_options is expected to save</span>
+<span class="sd"> high-level options, for instances, project and zone information, which</span>
+<span class="sd"> apply to all dataflow operators in the DAG.</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> For more detail on job submission have a look at the reference:</span>
+<span class="sd"> https://cloud.google.com/dataflow/pipelines/specifying-exec-params</span>
+
+<span class="sd"> :param py_file: Reference to the python dataflow pipleline file.py, e.g.,</span>
+<span class="sd"> /some/local/file/path/to/your/python/pipeline/file.</span>
+<span class="sd"> :type py_file: string</span>
+<span class="sd"> :param py_options: Additional python options.</span>
+<span class="sd"> :type pyt_options: list of strings, e.g., ["-m", "-v"].</span>
+<span class="sd"> :param dataflow_default_options: Map of default job options.</span>
+<span class="sd"> :type dataflow_default_options: dict</span>
+<span class="sd"> :param options: Map of job specific options.</span>
+<span class="sd"> :type options: dict</span>
+<span class="sd"> :param gcp_conn_id: The connection ID to use connecting to Google Cloud</span>
+<span class="sd"> Platform.</span>
+<span class="sd"> :type gcp_conn_id: string</span>
+<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd"> For this to work, the service account making the request must have</span>
+<span class="sd"> domain-wide delegation enabled.</span>
+<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> :param poll_sleep: The time in seconds to sleep between polling Google</span>
+<span class="sd"> Cloud Platform for the dataflow job status while the job is in the</span>
+<span class="sd"> JOB_STATE_RUNNING state.</span>
+<span class="sd"> :type poll_sleep: int</span>
+<span class="sd"> """</span>
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'options'</span><span class="p">,</span> <span class="s1">'dataflow_default_options'</span><span class="p">]</span>
@@ -428,38 +460,7 @@
<span class="n">poll_sleep</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span>
<span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Create a new DataFlowPythonOperator. Note that both</span>
-<span class="sd"> dataflow_default_options and options will be merged to specify pipeline</span>
-<span class="sd"> execution parameter, and dataflow_default_options is expected to save</span>
-<span class="sd"> high-level options, for instances, project and zone information, which</span>
-<span class="sd"> apply to all dataflow operators in the DAG.</span>
-
-<span class="sd"> .. seealso::</span>
-<span class="sd"> For more detail on job submission have a look at the reference:</span>
-<span class="sd"> https://cloud.google.com/dataflow/pipelines/specifying-exec-params</span>
-<span class="sd"> :param py_file: Reference to the python dataflow pipleline file.py, e.g.,</span>
-<span class="sd"> /some/local/file/path/to/your/python/pipeline/file.</span>
-<span class="sd"> :type py_file: string</span>
-<span class="sd"> :param py_options: Additional python options.</span>
-<span class="sd"> :type pyt_options: list of strings, e.g., ["-m", "-v"].</span>
-<span class="sd"> :param dataflow_default_options: Map of default job options.</span>
-<span class="sd"> :type dataflow_default_options: dict</span>
-<span class="sd"> :param options: Map of job specific options.</span>
-<span class="sd"> :type options: dict</span>
-<span class="sd"> :param gcp_conn_id: The connection ID to use connecting to Google Cloud</span>
-<span class="sd"> Platform.</span>
-<span class="sd"> :type gcp_conn_id: string</span>
-<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
-<span class="sd"> For this to work, the service account making the request must have</span>
-<span class="sd"> domain-wide delegation enabled.</span>
-<span class="sd"> :type delegate_to: string</span>
-<span class="sd"> :param poll_sleep: The time in seconds to sleep between polling Google</span>
-<span class="sd"> Cloud Platform for the dataflow job status while the job is in the</span>
-<span class="sd"> JOB_STATE_RUNNING state.</span>
-<span class="sd"> :type poll_sleep: int</span>
-<span class="sd"> """</span>
<span class="nb">super</span><span class="p">(</span><span class="n">DataFlowPythonOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">py_file</span> <span class="o">=</span> <span class="n">py_file</span>
@@ -519,21 +520,21 @@
<span class="c1"># Extracts bucket_id and object_id by first removing 'gs://' prefix and</span>
<span class="c1"># then split the remaining by path delimiter '/'.</span>
<span class="n">path_components</span> <span class="o">=</span> <span class="n">file_name</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">GCS_PREFIX_LENGTH</span><span class="p">:]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'/'</span><span class="p">)</span>
- <span class="k">if</span> <span class="n">path_components</span> <span class="o"><</span> <span class="mi">2</span><span class="p">:</span>
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">path_components</span><span class="p">)</span> <span class="o"><</span> <span class="mi">2</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
- <span class="s1">'Invalid Google Cloud Storage (GCS) object path: </span><span class="si">{}</span><span class="s1">.'</span>
+ <span class="s1">'Invalid Google Cloud Storage (GCS) object path: </span><span class="si">{}</span><span class="s1">'</span>
<span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">file_name</span><span class="p">))</span>
<span class="n">bucket_id</span> <span class="o">=</span> <span class="n">path_components</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">object_id</span> <span class="o">=</span> <span class="s1">'/'</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">path_components</span><span class="p">[</span><span class="mi">1</span><span class="p">:])</span>
<span class="n">local_file</span> <span class="o">=</span> <span class="s1">'/tmp/dataflow</span><span class="si">{}</span><span class="s1">-</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">uuid</span><span class="o">.</span><span class="n">uuid1</span><span class="p">())[:</span><span class="mi">8</span><span class="p">] [...]
<span class="n">path_components</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
- <span class="n">file_size</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_gcs_hook</span><span class="o">.</span><span class="n">download</span><span class="p">(</span><span class="n">bucket_id</span><span class="p">,</span> <span class="n">object_id</span><span class="p">,</span> <span class="n">local_file</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_gcs_hook</span><span class="o">.</span><span class="n">download</span><span class="p">(</span><span class="n">bucket_id</span><span class="p">,</span> <span class="n">object_id</span><span class="p">,</span> <span class="n">local_file</span><span class="p">)</span>
- <span class="k">if</span> <span class="n">file_size</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">local_file</span><span class="p">)</span><span class="o">.</span><span class="n">st_size</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
<span class="k">return</span> <span class="n">local_file</span>
<span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
- <span class="s1">'Failed to download Google Cloud Storage GCS object: </span><span class="si">{}</span><span class="s1">'</span>
+ <span class="s1">'Failed to download Google Cloud Storage (GCS) object: </span><span class="si">{}</span><span class="s1">'</span>
<span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">file_name</span><span class="p">))</span>
</pre></div>
@@ -565,20 +566,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/operators/dataproc_operator.html b/_modules/airflow/contrib/operators/dataproc_operator.html
index 6bc5d70..ec9caec 100644
--- a/_modules/airflow/contrib/operators/dataproc_operator.html
+++ b/_modules/airflow/contrib/operators/dataproc_operator.html
@@ -212,8 +212,9 @@
<span class="sd"> :type cluster_name: string</span>
<span class="sd"> :param project_id: The ID of the google cloud project in which</span>
<span class="sd"> to create the cluster. (templated)</span>
-<span class="sd"> :type project_id: string</span>
-<span class="sd"> :param num_workers: The # of workers to spin up</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :param num_workers: The # of workers to spin up. If set to zero will</span>
+<span class="sd"> spin up cluster in a single node mode</span>
<span class="sd"> :type num_workers: int</span>
<span class="sd"> :param storage_bucket: The storage bucket to use, setting to None lets dataproc</span>
<span class="sd"> generate a custom one for you</span>
@@ -229,6 +230,9 @@
<span class="sd"> :type metadata: dict</span>
<span class="sd"> :param image_version: the version of software inside the Dataproc cluster</span>
<span class="sd"> :type image_version: string</span>
+<span class="sd"> :param custom_image: custom Dataproc image for more info see</span>
+<span class="sd"> https://cloud.google.com/dataproc/docs/guides/dataproc-images</span>
+<span class="sd"> :type: custom_image: string</span>
<span class="sd"> :param properties: dict of properties to set on</span>
<span class="sd"> config files (e.g. spark-defaults.conf), see</span>
<span class="sd"> https://cloud.google.com/dataproc/docs/reference/rest/v1/ \</span>
@@ -236,10 +240,20 @@
<span class="sd"> :type properties: dict</span>
<span class="sd"> :param master_machine_type: Compute engine machine type to use for the master node</span>
<span class="sd"> :type master_machine_type: string</span>
+<span class="sd"> :param master_disk_type: Type of the boot disk for the master node</span>
+<span class="sd"> (default is ``pd-standard``).</span>
+<span class="sd"> Valid values: ``pd-ssd`` (Persistent Disk Solid State Drive) or</span>
+<span class="sd"> ``pd-standard`` (Persistent Disk Hard Disk Drive).</span>
+<span class="sd"> :type master_disk_type: string</span>
<span class="sd"> :param master_disk_size: Disk size for the master node</span>
<span class="sd"> :type master_disk_size: int</span>
<span class="sd"> :param worker_machine_type: Compute engine machine type to use for the worker nodes</span>
<span class="sd"> :type worker_machine_type: string</span>
+<span class="sd"> :param worker_disk_type: Type of the boot disk for the worker node</span>
+<span class="sd"> (default is ``pd-standard``).</span>
+<span class="sd"> Valid values: ``pd-ssd`` (Persistent Disk Solid State Drive) or</span>
+<span class="sd"> ``pd-standard`` (Persistent Disk Hard Disk Drive).</span>
+<span class="sd"> :type worker_disk_type: string</span>
<span class="sd"> :param worker_disk_size: Disk size for the worker nodes</span>
<span class="sd"> :type worker_disk_size: int</span>
<span class="sd"> :param num_preemptible_workers: The # of preemptible worker nodes to spin up</span>
@@ -299,11 +313,14 @@
<span class="n">init_actions_uris</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">init_action_timeout</span><span class="o">=</span><span class="s2">"10m"</span><span class="p">,</span>
<span class="n">metadata</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">custom_image</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">image_version</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">properties</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">master_machine_type</span><span class="o">=</span><span class="s1">'n1-standard-4'</span><span class="p">,</span>
+ <span class="n">master_disk_type</span><span class="o">=</span><span class="s1">'pd-standard'</span><span class="p">,</span>
<span class="n">master_disk_size</span><span class="o">=</span><span class="mi">500</span><span class="p">,</span>
<span class="n">worker_machine_type</span><span class="o">=</span><span class="s1">'n1-standard-4'</span><span class="p">,</span>
+ <span class="n">worker_disk_type</span><span class="o">=</span><span class="s1">'pd-standard'</span><span class="p">,</span>
<span class="n">worker_disk_size</span><span class="o">=</span><span class="mi">500</span><span class="p">,</span>
<span class="n">num_preemptible_workers</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
<span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -329,11 +346,14 @@
<span class="bp">self</span><span class="o">.</span><span class="n">init_actions_uris</span> <span class="o">=</span> <span class="n">init_actions_uris</span>
<span class="bp">self</span><span class="o">.</span><span class="n">init_action_timeout</span> <span class="o">=</span> <span class="n">init_action_timeout</span>
<span class="bp">self</span><span class="o">.</span><span class="n">metadata</span> <span class="o">=</span> <span class="n">metadata</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">custom_image</span> <span class="o">=</span> <span class="n">custom_image</span>
<span class="bp">self</span><span class="o">.</span><span class="n">image_version</span> <span class="o">=</span> <span class="n">image_version</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">properties</span> <span class="o">=</span> <span class="n">properties</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">properties</span> <span class="o">=</span> <span class="n">properties</span> <span class="ow">or</span> <span class="nb">dict</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">master_machine_type</span> <span class="o">=</span> <span class="n">master_machine_type</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">master_disk_type</span> <span class="o">=</span> <span class="n">master_disk_type</span>
<span class="bp">self</span><span class="o">.</span><span class="n">master_disk_size</span> <span class="o">=</span> <span class="n">master_disk_size</span>
<span class="bp">self</span><span class="o">.</span><span class="n">worker_machine_type</span> <span class="o">=</span> <span class="n">worker_machine_type</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">worker_disk_type</span> <span class="o">=</span> <span class="n">worker_disk_type</span>
<span class="bp">self</span><span class="o">.</span><span class="n">worker_disk_size</span> <span class="o">=</span> <span class="n">worker_disk_size</span>
<span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="o">=</span> <span class="n">labels</span>
<span class="bp">self</span><span class="o">.</span><span class="n">zone</span> <span class="o">=</span> <span class="n">zone</span>
@@ -347,6 +367,19 @@
<span class="bp">self</span><span class="o">.</span><span class="n">idle_delete_ttl</span> <span class="o">=</span> <span class="n">idle_delete_ttl</span>
<span class="bp">self</span><span class="o">.</span><span class="n">auto_delete_time</span> <span class="o">=</span> <span class="n">auto_delete_time</span>
<span class="bp">self</span><span class="o">.</span><span class="n">auto_delete_ttl</span> <span class="o">=</span> <span class="n">auto_delete_ttl</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">single_node</span> <span class="o">=</span> <span class="n">num_workers</span> <span class="o">==</span> <span class="mi">0</span>
+
+ <span class="k">assert</span> <span class="ow">not</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">custom_image</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">image_version</span><span class="p">),</span> \
+ <span class="s2">"custom_image and image_version can't be both set"</span>
+
+ <span class="k">assert</span> <span class="p">(</span>
+ <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">single_node</span> <span class="ow">or</span> <span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">single_node</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_preemptible_workers</span> <span class="o">==</span> <span class="mi">0</span>
+ <span class="p">)</span>
+ <span class="p">),</span> <span class="s2">"num_workers == 0 means single node mode - no preemptibles allowed"</span>
+
+ <span class="k">assert</span> <span class="ow">not</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">custom_image</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">image_version</span><span class="p">),</span> \
+ <span class="s2">"custom_image and image_version can't be both set"</span>
<span class="k">def</span> <span class="nf">_get_cluster_list_for_project</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">service</span><span class="p">):</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">service</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">regions</span><span class="p">()</span><span class="o">.</span><span class="n">clusters</span><span class="p">()</span><span class="o">.</span><span class="n">list</span><span class="p">(</span>
@@ -433,6 +466,7 @@
<span class="s1">'numInstances'</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
<span class="s1">'machineTypeUri'</span><span class="p">:</span> <span class="n">master_type_uri</span><span class="p">,</span>
<span class="s1">'diskConfig'</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s1">'bootDiskType'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">master_disk_type</span><span class="p">,</span>
<span class="s1">'bootDiskSizeGb'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">master_disk_size</span>
<span class="p">}</span>
<span class="p">},</span>
@@ -440,6 +474,7 @@
<span class="s1">'numInstances'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span><span class="p">,</span>
<span class="s1">'machineTypeUri'</span><span class="p">:</span> <span class="n">worker_type_uri</span><span class="p">,</span>
<span class="s1">'diskConfig'</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s1">'bootDiskType'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_disk_type</span><span class="p">,</span>
<span class="s1">'bootDiskSizeGb'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_disk_size</span>
<span class="p">}</span>
<span class="p">},</span>
@@ -453,6 +488,7 @@
<span class="s1">'numInstances'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_preemptible_workers</span><span class="p">,</span>
<span class="s1">'machineTypeUri'</span><span class="p">:</span> <span class="n">worker_type_uri</span><span class="p">,</span>
<span class="s1">'diskConfig'</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s1">'bootDiskType'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_disk_type</span><span class="p">,</span>
<span class="s1">'bootDiskSizeGb'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_disk_size</span>
<span class="p">},</span>
<span class="s1">'isPreemptible'</span><span class="p">:</span> <span class="kc">True</span>
@@ -482,6 +518,17 @@
<span class="n">cluster_data</span><span class="p">[</span><span class="s1">'config'</span><span class="p">][</span><span class="s1">'gceClusterConfig'</span><span class="p">][</span><span class="s1">'tags'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">tags</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">image_version</span><span class="p">:</span>
<span class="n">cluster_data</span><span class="p">[</span><span class="s1">'config'</span><span class="p">][</span><span class="s1">'softwareConfig'</span><span class="p">][</span><span class="s1">'imageVersion'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">image_version</span>
+ <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">custom_image</span><span class="p">:</span>
+ <span class="n">custom_image_url</span> <span class="o">=</span> <span class="s1">'https://www.googleapis.com/compute/beta/projects/'</span> \
+ <span class="s1">'</span><span class="si">{}</span><span class="s1">/global/images/</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">custom_image</span><span class="p">)</span>
+ <span class="n">cluster_data</span><span class="p">[</span><span class="s1">'config'</span><span class="p">][</span><span class="s1">'masterConfig'</span><span class="p">][</span><span class="s1">'imageUri'</span><span class="p">]</span> <span class="o">=</span> <span class="n">custom_image_url</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">single_node</span><span class="p">:</span>
+ <span class="n">cluster_data</span><span class="p">[</span><span class="s1">'config'</span><span class="p">][</span><span class="s1">'workerConfig'</span><span class="p">][</span><span class="s1">'imageUri'</span><span class="p">]</span> <span class="o">=</span> <span class="n">custom_image_url</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">single_node</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">properties</span><span class="p">[</span><span class="s2">"dataproc:dataproc.allow.zero.workers"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"true"</span>
+
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">properties</span><span class="p">:</span>
<span class="n">cluster_data</span><span class="p">[</span><span class="s1">'config'</span><span class="p">][</span><span class="s1">'softwareConfig'</span><span class="p">][</span><span class="s1">'properties'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">properties</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">idle_delete_ttl</span><span class="p">:</span>
@@ -510,7 +557,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">service_account_scopes</span>
<span class="k">return</span> <span class="n">cluster_data</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="DataprocClusterCreateOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataprocClusterCreateOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Creating cluster: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span><span class="p">)</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span>
<span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
@@ -546,7 +593,7 @@
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">e</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_done</span><span class="p">(</span><span class="n">service</span><span class="p">)</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_done</span><span class="p">(</span><span class="n">service</span><span class="p">)</span></div></div>
<div class="viewcode-block" id="DataprocClusterScaleOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataprocClusterScaleOperator">[docs]</a><span class="k">class</span> <span class="nc">DataprocClusterScaleOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
@@ -556,14 +603,14 @@
<span class="sd"> **Example**: ::</span>
-<span class="sd"> t1 = DataprocClusterScaleOperator(</span>
-<span class="sd"> task_id='dataproc_scale',</span>
-<span class="sd"> project_id='my-project',</span>
-<span class="sd"> cluster_name='cluster-1',</span>
-<span class="sd"> num_workers=10,</span>
-<span class="sd"> num_preemptible_workers=10,</span>
-<span class="sd"> graceful_decommission_timeout='1h'</span>
-<span class="sd"> dag=dag)</span>
+<span class="sd"> t1 = DataprocClusterScaleOperator(</span>
+<span class="sd"> task_id='dataproc_scale',</span>
+<span class="sd"> project_id='my-project',</span>
+<span class="sd"> cluster_name='cluster-1',</span>
+<span class="sd"> num_workers=10,</span>
+<span class="sd"> num_preemptible_workers=10,</span>
+<span class="sd"> graceful_decommission_timeout='1h',</span>
+<span class="sd"> dag=dag)</span>
<span class="sd"> .. seealso::</span>
<span class="sd"> For more detail on about scaling clusters have a look at the reference:</span>
@@ -672,7 +719,7 @@
<span class="s2">" should be expressed in day, hours, minutes or seconds. "</span>
<span class="s2">" i.e. 1d, 4h, 10m, 30s"</span><span class="p">)</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="DataprocClusterScaleOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataprocClusterScaleOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Scaling cluster: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span><span class="p">)</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span>
<span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
@@ -694,7 +741,7 @@
<span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
<span class="n">operation_name</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Cluster scale operation name: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_done</span><span class="p">(</span><span class="n">service</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">)</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_done</span><span class="p">(</span><span class="n">service</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">)</span></div></div>
<div class="viewcode-block" id="DataprocClusterDeleteOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataprocClusterDeleteOperator">[docs]</a><span class="k">class</span> <span class="nc">DataprocClusterDeleteOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
@@ -750,7 +797,7 @@
<span class="k">return</span>
<span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">15</span><span class="p">)</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="DataprocClusterDeleteOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataprocClusterDeleteOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Deleting cluster: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span><span class="p">)</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span>
<span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
@@ -765,7 +812,7 @@
<span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
<span class="n">operation_name</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Cluster delete operation name: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_done</span><span class="p">(</span><span class="n">service</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">)</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_done</span><span class="p">(</span><span class="n">service</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">)</span></div></div>
<div class="viewcode-block" id="DataProcPigOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcPigOperator">[docs]</a><span class="k">class</span> <span class="nc">DataProcPigOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
@@ -863,7 +910,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span> <span class="o">=</span> <span class="n">dataproc_pig_jars</span>
<span class="bp">self</span><span class="o">.</span><span class="n">region</span> <span class="o">=</span> <span class="n">region</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="DataProcPigOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcPigOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
<span class="n">job</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">create_job_template</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span><span class="p">,</span> <span class="s2">"pigJob"</span><span class="p">,</span>
@@ -877,7 +924,7 @@
<span class="n">job</span><span class="o">.</span><span class="n">add_jar_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span><span class="p">)</span>
<span class="n">job</span><span class="o">.</span><span class="n">set_job_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">)</span>
- <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div>
+ <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div></div>
<div class="viewcode-block" id="DataProcHiveOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcHiveOperator">[docs]</a><span class="k">class</span> <span class="nc">DataProcHiveOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
@@ -943,7 +990,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span> <span class="o">=</span> <span class="n">dataproc_hive_jars</span>
<span class="bp">self</span><span class="o">.</span><span class="n">region</span> <span class="o">=</span> <span class="n">region</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="DataProcHiveOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcHiveOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
@@ -958,7 +1005,7 @@
<span class="n">job</span><span class="o">.</span><span class="n">add_jar_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span><span class="p">)</span>
<span class="n">job</span><span class="o">.</span><span class="n">set_job_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">)</span>
- <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div>
+ <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div></div>
<div class="viewcode-block" id="DataProcSparkSqlOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcSparkSqlOperator">[docs]</a><span class="k">class</span> <span class="nc">DataProcSparkSqlOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
@@ -1025,7 +1072,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span> <span class="o">=</span> <span class="n">dataproc_spark_jars</span>
<span class="bp">self</span><span class="o">.</span><span class="n">region</span> <span class="o">=</span> <span class="n">region</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="DataProcSparkSqlOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcSparkSqlOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
@@ -1040,7 +1087,7 @@
<span class="n">job</span><span class="o">.</span><span class="n">add_jar_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span><span class="p">)</span>
<span class="n">job</span><span class="o">.</span><span class="n">set_job_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">)</span>
- <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div>
+ <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div></div>
<div class="viewcode-block" id="DataProcSparkOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcSparkOperator">[docs]</a><span class="k">class</span> <span class="nc">DataProcSparkOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
@@ -1118,7 +1165,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span> <span class="o">=</span> <span class="n">dataproc_spark_jars</span>
<span class="bp">self</span><span class="o">.</span><span class="n">region</span> <span class="o">=</span> <span class="n">region</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="DataProcSparkOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcSparkOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
<span class="n">job</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">create_job_template</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span><span class="p">,</span> <span class="s2">"sparkJob"</span><span class="p">,</span>
@@ -1131,7 +1178,7 @@
<span class="n">job</span><span class="o">.</span><span class="n">add_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">files</span><span class="p">)</span>
<span class="n">job</span><span class="o">.</span><span class="n">set_job_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">)</span>
- <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div>
+ <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div></div>
<div class="viewcode-block" id="DataProcHadoopOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcHadoopOperator">[docs]</a><span class="k">class</span> <span class="nc">DataProcHadoopOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
@@ -1209,7 +1256,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span> <span class="o">=</span> <span class="n">dataproc_hadoop_jars</span>
<span class="bp">self</span><span class="o">.</span><span class="n">region</span> <span class="o">=</span> <span class="n">region</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="DataProcHadoopOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcHadoopOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
<span class="n">job</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">create_job_template</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span><span class="p">,</span> <span class="s2">"hadoopJob"</span><span class="p">,</span>
@@ -1222,7 +1269,7 @@
<span class="n">job</span><span class="o">.</span><span class="n">add_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">files</span><span class="p">)</span>
<span class="n">job</span><span class="o">.</span><span class="n">set_job_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">)</span>
- <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div>
+ <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div></div>
<div class="viewcode-block" id="DataProcPySparkOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcPySparkOperator">[docs]</a><span class="k">class</span> <span class="nc">DataProcPySparkOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
@@ -1327,7 +1374,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span> <span class="o">=</span> <span class="n">dataproc_pyspark_jars</span>
<span class="bp">self</span><span class="o">.</span><span class="n">region</span> <span class="o">=</span> <span class="n">region</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="DataProcPySparkOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcPySparkOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span>
<span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span>
@@ -1353,7 +1400,7 @@
<span class="n">job</span><span class="o">.</span><span class="n">add_python_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">pyfiles</span><span class="p">)</span>
<span class="n">job</span><span class="o">.</span><span class="n">set_job_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">)</span>
- <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div>
+ <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div></div>
<div class="viewcode-block" id="DataprocWorkflowTemplateBaseOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.dataproc_operator.DataprocWorkflowTemplateBaseOperator">[docs]</a><span class="k">class</span> <span class="nc">DataprocWorkflowTemplateBaseOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
@@ -1376,8 +1423,8 @@
<span class="n">api_version</span><span class="o">=</span><span class="s1">'v1beta2'</span>
<span class="p">)</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">wait</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">start</span><span class="p">())</span>
+<div class="viewcode-block" id="DataprocWorkflowTemplateBaseOperator.execute"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.dataproc_operator.DataprocWorkflowTemplateBaseOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">wait</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">start</span><span class="p">())</span></div>
<span class="k">def</span> <span class="nf">start</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'plese start a workflow operation'</span><span class="p">)</span></div>
@@ -1498,20 +1545,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/operators/datastore_export_operator.html b/_modules/airflow/contrib/operators/datastore_export_operator.html
index 915c753..2237184 100644
--- a/_modules/airflow/contrib/operators/datastore_export_operator.html
+++ b/_modules/airflow/contrib/operators/datastore_export_operator.html
@@ -245,7 +245,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">overwrite_existing</span> <span class="o">=</span> <span class="n">overwrite_existing</span>
<span class="bp">self</span><span class="o">.</span><span class="n">xcom_push</span> <span class="o">=</span> <span class="n">xcom_push</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="DatastoreExportOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.datastore_export_operator.DatastoreExportOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Exporting data to Cloud Storage bucket '</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">overwrite_existing</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">namespace</span><span class="p">:</span>
@@ -268,7 +268,7 @@
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Operation failed: result=</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">result</span><span class="p">))</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">xcom_push</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">result</span></div>
+ <span class="k">return</span> <span class="n">result</span></div></div>
</pre></div>
</div>
@@ -299,20 +299,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/operators/datastore_import_operator.html b/_modules/airflow/contrib/operators/datastore_import_operator.html
index c7d84b4..8feddfb 100644
--- a/_modules/airflow/contrib/operators/datastore_import_operator.html
+++ b/_modules/airflow/contrib/operators/datastore_import_operator.html
@@ -239,7 +239,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">polling_interval_in_seconds</span> <span class="o">=</span> <span class="n">polling_interval_in_seconds</span>
<span class="bp">self</span><span class="o">.</span><span class="n">xcom_push</span> <span class="o">=</span> <span class="n">xcom_push</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="DatastoreImportOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.datastore_import_operator.DatastoreImportOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Importing data from Cloud Storage bucket </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">)</span>
<span class="n">ds_hook</span> <span class="o">=</span> <span class="n">DatastoreHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">datastore_conn_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">ds_hook</span><span class="o">.</span><span class="n">import_from_storage_bucket</span><span class="p">(</span><span class="n">bucket</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span>
@@ -256,7 +256,7 @@
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Operation failed: result=</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">result</span><span class="p">))</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">xcom_push</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">result</span></div>
+ <span class="k">return</span> <span class="n">result</span></div></div>
</pre></div>
</div>
@@ -287,20 +287,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/operators/discord_webhook_operator.html b/_modules/airflow/contrib/operators/discord_webhook_operator.html
index 3122f2a..c480587 100644
--- a/_modules/airflow/contrib/operators/discord_webhook_operator.html
+++ b/_modules/airflow/contrib/operators/discord_webhook_operator.html
@@ -287,20 +287,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/operators/druid_operator.html b/_modules/airflow/contrib/operators/druid_operator.html
index dd9de81..7941dc6 100644
--- a/_modules/airflow/contrib/operators/druid_operator.html
+++ b/_modules/airflow/contrib/operators/druid_operator.html
@@ -214,13 +214,13 @@
<span class="n">separators</span><span class="o">=</span><span class="p">(</span><span class="s1">','</span><span class="p">,</span> <span class="s1">': '</span><span class="p">)</span>
<span class="p">)</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="DruidOperator.execute"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.druid_operator.DruidOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">DruidHook</span><span class="p">(</span>
<span class="n">druid_ingest_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">,</span>
<span class="n">max_ingestion_time</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">max_ingestion_time</span>
<span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Sumitting </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">index_spec_str</span><span class="p">)</span>
- <span class="n">hook</span><span class="o">.</span><span class="n">submit_indexing_job</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">index_spec_str</span><span class="p">)</span></div>
+ <span class="n">hook</span><span class="o">.</span><span class="n">submit_indexing_job</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">index_spec_str</span><span class="p">)</span></div></div>
</pre></div>
</div>
@@ -251,20 +251,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/operators/ecs_operator.html b/_modules/airflow/contrib/operators/ecs_operator.html
index becc9fb..e188d1c 100644
--- a/_modules/airflow/contrib/operators/ecs_operator.html
+++ b/_modules/airflow/contrib/operators/ecs_operator.html
@@ -178,6 +178,7 @@
<span class="c1"># specific language governing permissions and limitations</span>
<span class="c1"># under the License.</span>
<span class="kn">import</span> <span class="nn">sys</span>
+<span class="kn">import</span> <span class="nn">re</span>
<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
@@ -194,17 +195,18 @@
<span class="sd"> :type task_definition: str</span>
<span class="sd"> :param cluster: the cluster name on EC2 Container Service</span>
<span class="sd"> :type cluster: str</span>
-<span class="sd"> :param: overrides: the same parameter that boto3 will receive (templated):</span>
-<span class="sd"> http://boto3.readthedocs.org/en/latest/reference/services/ecs.html#ECS.Client.run_task</span>
-<span class="sd"> :type: overrides: dict</span>
+<span class="sd"> :param overrides: the same parameter that boto3 will receive (templated):</span>
+<span class="sd"> http://boto3.readthedocs.org/en/latest/reference/services/ecs.html#ECS.Client.run_task</span>
+<span class="sd"> :type overrides: dict</span>
<span class="sd"> :param aws_conn_id: connection id of AWS credentials / region name. If None,</span>
-<span class="sd"> credential boto3 strategy will be used</span>
-<span class="sd"> (http://boto3.readthedocs.io/en/latest/guide/configuration.html).</span>
+<span class="sd"> credential boto3 strategy will be used</span>
+<span class="sd"> (http://boto3.readthedocs.io/en/latest/guide/configuration.html).</span>
<span class="sd"> :type aws_conn_id: str</span>
<span class="sd"> :param region_name: region name to use in AWS Hook.</span>
<span class="sd"> Override the region_name in connection (if provided)</span>
+<span class="sd"> :type region_name: str</span>
<span class="sd"> :param launch_type: the launch type on which to run your task ('EC2' or 'FARGATE')</span>
-<span class="sd"> :type: launch_type: str</span>
+<span class="sd"> :type launch_type: str</span>
<span class="sd"> """</span>
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f0ede4'</span>
@@ -226,7 +228,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="ECSOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.ecs_operator.ECSOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
<span class="s1">'Running ECS Task - Task definition: </span><span class="si">%s</span><span class="s1"> - on cluster </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_definition</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster</span>
@@ -255,7 +257,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_task_ended</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_check_success_task</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'ECS Task has been successfully executed: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">response</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'ECS Task has been successfully executed: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">response</span><span class="p">)</span></div>
<span class="k">def</span> <span class="nf">_wait_for_task_ended</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">waiter</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">get_waiter</span><span class="p">(</span><span class="s1">'tasks_stopped'</span><span class="p">)</span>
@@ -276,6 +278,15 @@
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
<span class="k">for</span> <span class="n">task</span> <span class="ow">in</span> <span class="n">response</span><span class="p">[</span><span class="s1">'tasks'</span><span class="p">]:</span>
+ <span class="c1"># This is a `stoppedReason` that indicates a task has not</span>
+ <span class="c1"># successfully finished, but there is no other indication of failure</span>
+ <span class="c1"># in the response.</span>
+ <span class="c1"># See, https://docs.aws.amazon.com/AmazonECS/latest/developerguide/stopped-task-errors.html # noqa E501</span>
+ <span class="k">if</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="sa">r</span><span class="s1">'Host EC2 \(instance .+?\) (stopped|terminated)\.'</span><span class="p">,</span>
+ <span class="n">task</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'stoppedReason'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)):</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'The task was stopped because the host instance terminated: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span>
+ <span class="nb">format</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'stoppedReason'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)))</span>
<span class="n">containers</span> <span class="o">=</span> <span class="n">task</span><span class="p">[</span><span class="s1">'containers'</span><span class="p">]</span>
<span class="k">for</span> <span class="n">container</span> <span class="ow">in</span> <span class="n">containers</span><span class="p">:</span>
<span class="k">if</span> <span class="n">container</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'lastStatus'</span><span class="p">)</span> <span class="o">==</span> <span class="s1">'STOPPED'</span> <span class="ow">and</span> \
@@ -294,12 +305,12 @@
<span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span>
<span class="p">)</span>
- <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<div class="viewcode-block" id="ECSOperator.on_kill"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.ecs_operator.ECSOperator.on_kill">[docs]</a> <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">stop_task</span><span class="p">(</span>
<span class="n">cluster</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cluster</span><span class="p">,</span>
<span class="n">task</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">arn</span><span class="p">,</span>
<span class="n">reason</span><span class="o">=</span><span class="s1">'Task killed by the user'</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">response</span><span class="p">)</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">response</span><span class="p">)</span></div></div>
</pre></div>
</div>
@@ -330,20 +341,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/operators/emr_add_steps_operator.html b/_modules/airflow/contrib/operators/emr_add_steps_operator.html
index acb9a63..981d542 100644
--- a/_modules/airflow/contrib/operators/emr_add_steps_operator.html
+++ b/_modules/airflow/contrib/operators/emr_add_steps_operator.html
@@ -211,7 +211,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span> <span class="o">=</span> <span class="n">aws_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">steps</span> <span class="o">=</span> <span class="n">steps</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="EmrAddStepsOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.emr_add_steps_operator.EmrAddStepsOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">emr</span> <span class="o">=</span> <span class="n">EmrHook</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Adding steps to </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_flow_id</span><span class="p">)</span>
@@ -221,7 +221,7 @@
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Adding steps failed: </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">response</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Steps </span><span class="si">%s</span><span class="s1"> added to JobFlow'</span><span class="p">,</span> <span class="n">response</span><span class="p">[</span><span class="s1">'StepIds'</span><span class="p">])</span>
- <span class="k">return</span> <span class="n">response</span><span class="p">[</span><span class="s1">'StepIds'</span><span class="p">]</span></div>
+ <span class="k">return</span> <span class="n">response</span><span class="p">[</span><span class="s1">'StepIds'</span><span class="p">]</span></div></div>
</pre></div>
</div>
@@ -252,20 +252,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/operators/emr_create_job_flow_operator.html b/_modules/airflow/contrib/operators/emr_create_job_flow_operator.html
index 21d48ff..fa067fa 100644
--- a/_modules/airflow/contrib/operators/emr_create_job_flow_operator.html
+++ b/_modules/airflow/contrib/operators/emr_create_job_flow_operator.html
@@ -215,7 +215,7 @@
<span class="n">job_flow_overrides</span> <span class="o">=</span> <span class="p">{}</span>
<span class="bp">self</span><span class="o">.</span><span class="n">job_flow_overrides</span> <span class="o">=</span> <span class="n">job_flow_overrides</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="EmrCreateJobFlowOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.emr_create_job_flow_operator.EmrCreateJobFlowOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">emr</span> <span class="o">=</span> <span class="n">EmrHook</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">,</span> <span class="n">emr_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">emr_conn_id</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
@@ -228,7 +228,7 @@
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'JobFlow creation failed: </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">response</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'JobFlow with id </span><span class="si">%s</span><span class="s1"> created'</span><span class="p">,</span> <span class="n">response</span><span class="p">[</span><span class="s1">'JobFlowId'</span><span class="p">])</span>
- <span class="k">return</span> <span class="n">response</span><span class="p">[</span><span class="s1">'JobFlowId'</span><span class="p">]</span></div>
+ <span class="k">return</span> <span class="n">response</span><span class="p">[</span><span class="s1">'JobFlowId'</span><span class="p">]</span></div></div>
</pre></div>
</div>
@@ -259,20 +259,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/operators/emr_terminate_job_flow_operator.html b/_modules/airflow/contrib/operators/emr_terminate_job_flow_operator.html
index 82141a7..a325633 100644
--- a/_modules/airflow/contrib/operators/emr_terminate_job_flow_operator.html
+++ b/_modules/airflow/contrib/operators/emr_terminate_job_flow_operator.html
@@ -206,7 +206,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">job_flow_id</span> <span class="o">=</span> <span class="n">job_flow_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span> <span class="o">=</span> <span class="n">aws_conn_id</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="EmrTerminateJobFlowOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.emr_terminate_job_flow_operator.EmrTerminateJobFlowOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">emr</span> <span class="o">=</span> <span class="n">EmrHook</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Terminating JobFlow </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_flow_id</span><span class="p">)</span>
@@ -215,7 +215,7 @@
<span class="k">if</span> <span class="ow">not</span> <span class="n">response</span><span class="p">[</span><span class="s1">'ResponseMetadata'</span><span class="p">][</span><span class="s1">'HTTPStatusCode'</span><span class="p">]</span> <span class="o">==</span> <span class="mi">200</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'JobFlow termination failed: </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">response</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'JobFlow with id </span><span class="si">%s</span><span class="s1"> terminated'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_flow_id</span><span class="p">)</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'JobFlow with id </span><span class="si">%s</span><span class="s1"> terminated'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_flow_id</span><span class="p">)</span></div></div>
</pre></div>
</div>
@@ -246,20 +246,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/operators/file_to_gcs.html b/_modules/airflow/contrib/operators/file_to_gcs.html
index 0e67133..b763c19 100644
--- a/_modules/airflow/contrib/operators/file_to_gcs.html
+++ b/_modules/airflow/contrib/operators/file_to_gcs.html
@@ -186,7 +186,8 @@
<div class="viewcode-block" id="FileToGoogleCloudStorageOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.file_to_gcs.FileToGoogleCloudStorageOperator">[docs]</a><span class="k">class</span> <span class="nc">FileToGoogleCloudStorageOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
<span class="sd">"""</span>
-<span class="sd"> Uploads a file to Google Cloud Storage</span>
+<span class="sd"> Uploads a file to Google Cloud Storage.</span>
+<span class="sd"> Optionally can compress the file for upload.</span>
<span class="sd"> :param src: Path to the local file. (templated)</span>
<span class="sd"> :type src: string</span>
@@ -199,7 +200,9 @@
<span class="sd"> :param mime_type: The mime-type string</span>
<span class="sd"> :type mime_type: string</span>
<span class="sd"> :param delegate_to: The account to impersonate, if any</span>
-<span class="sd"> :type delegate_to: string</span>
+<span class="sd"> :type delegate_to: str</span>
+<span class="sd"> :param gzip: Allows for file to be compressed and uploaded as gzip</span>
+<span class="sd"> :type gzip: bool</span>
<span class="sd"> """</span>
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'src'</span><span class="p">,</span> <span class="s1">'dst'</span><span class="p">,</span> <span class="s1">'bucket'</span><span class="p">)</span>
@@ -211,6 +214,7 @@
<span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
<span class="n">mime_type</span><span class="o">=</span><span class="s1">'application/octet-stream'</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">gzip</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span>
<span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">FileToGoogleCloudStorageOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
@@ -220,6 +224,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span> <span class="o">=</span> <span class="n">google_cloud_storage_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">mime_type</span> <span class="o">=</span> <span class="n">mime_type</span>
<span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">gzip</span> <span class="o">=</span> <span class="n">gzip</span>
<div class="viewcode-block" id="FileToGoogleCloudStorageOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.file_to_gcs.FileToGoogleCloudStorageOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="sd">"""</span>
@@ -233,7 +238,9 @@
<span class="n">bucket</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span>
<span class="nb">object</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dst</span><span class="p">,</span>
<span class="n">mime_type</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mime_type</span><span class="p">,</span>
- <span class="n">filename</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">src</span><span class="p">)</span></div></div>
+ <span class="n">filename</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">src</span><span class="p">,</span>
+ <span class="n">gzip</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gzip</span><span class="p">,</span>
+ <span class="p">)</span></div></div>
</pre></div>
</div>
@@ -264,20 +271,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/operators/file_to_wasb.html b/_modules/airflow/contrib/operators/file_to_wasb.html
index 8a060ef..31d44c5 100644
--- a/_modules/airflow/contrib/operators/file_to_wasb.html
+++ b/_modules/airflow/contrib/operators/file_to_wasb.html
@@ -253,20 +253,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/operators/gcp_compute_operator.html b/_modules/airflow/contrib/operators/gcp_compute_operator.html
new file mode 100644
index 0000000..8aa8505
--- /dev/null
+++ b/_modules/airflow/contrib/operators/gcp_compute_operator.html
@@ -0,0 +1,394 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.gcp_compute_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.gcp_compute_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.gcp_compute_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_compute_hook</span> <span class="k">import</span> <span class="n">GceHook</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.utils.gcp_field_validator</span> <span class="k">import</span> <span class="n">GcpBodyFieldValidator</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<span class="k">class</span> <span class="nc">GceBaseOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Abstract base operator for Google Compute Engine operators to inherit from.</span>
+<span class="sd"> """</span>
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span>
+ <span class="n">zone</span><span class="p">,</span>
+ <span class="n">resource_id</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="o">=</span><span class="s1">'v1'</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">zone</span> <span class="o">=</span> <span class="n">zone</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">full_location</span> <span class="o">=</span> <span class="s1">'projects/</span><span class="si">{}</span><span class="s1">/zones/</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">zone</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">resource_id</span> <span class="o">=</span> <span class="n">resource_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">api_version</span> <span class="o">=</span> <span class="n">api_version</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_validate_inputs</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span> <span class="o">=</span> <span class="n">GceHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="n">api_version</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">api_version</span><span class [...]
+ <span class="nb">super</span><span class="p">(</span><span class="n">GceBaseOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_validate_inputs</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"The required parameter 'project_id' is missing"</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">zone</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"The required parameter 'zone' is missing"</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">resource_id</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"The required parameter 'resource_id' is missing"</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="k">pass</span>
+
+
+<div class="viewcode-block" id="GceInstanceStartOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcp_compute_operator.GceInstanceStartOperator">[docs]</a><span class="k">class</span> <span class="nc">GceInstanceStartOperator</span><span class="p">(</span><span class="n">GceBaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Start an instance in Google Compute Engine.</span>
+
+<span class="sd"> :param project_id: Google Cloud Platform project where the Compute Engine</span>
+<span class="sd"> instance exists.</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :param zone: Google Cloud Platform zone where the instance exists.</span>
+<span class="sd"> :type zone: str</span>
+<span class="sd"> :param resource_id: Name of the Compute Engine instance resource.</span>
+<span class="sd"> :type resource_id: str</span>
+<span class="sd"> :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: str</span>
+<span class="sd"> :param api_version: API version used (e.g. v1).</span>
+<span class="sd"> :type api_version: str</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'zone'</span><span class="p">,</span> <span class="s1">'resource_id'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span>
+ <span class="n">zone</span><span class="p">,</span>
+ <span class="n">resource_id</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="o">=</span><span class="s1">'v1'</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">GceInstanceStartOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+ <span class="n">project_id</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span> <span class="n">zone</span><span class="o">=</span><span class="n">zone</span><span class="p">,</span> <span class="n">resource_id</span><span class="o">=</span><span class="n">resource_id</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="n">api_version</span><span class="o">=</span><span class="n">api_version</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+<div class="viewcode-block" id="GceInstanceStartOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcp_compute_operator.GceInstanceStartOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span><span class="o">.</span><span class="n">start_instance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">zone</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">resourc [...]
+
+
+<div class="viewcode-block" id="GceInstanceStopOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcp_compute_operator.GceInstanceStopOperator">[docs]</a><span class="k">class</span> <span class="nc">GceInstanceStopOperator</span><span class="p">(</span><span class="n">GceBaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Stop an instance in Google Compute Engine.</span>
+
+<span class="sd"> :param project_id: Google Cloud Platform project where the Compute Engine</span>
+<span class="sd"> instance exists.</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :param zone: Google Cloud Platform zone where the instance exists.</span>
+<span class="sd"> :type zone: str</span>
+<span class="sd"> :param resource_id: Name of the Compute Engine instance resource.</span>
+<span class="sd"> :type resource_id: str</span>
+<span class="sd"> :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: str</span>
+<span class="sd"> :param api_version: API version used (e.g. v1).</span>
+<span class="sd"> :type api_version: str</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'zone'</span><span class="p">,</span> <span class="s1">'resource_id'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span>
+ <span class="n">zone</span><span class="p">,</span>
+ <span class="n">resource_id</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="o">=</span><span class="s1">'v1'</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">GceInstanceStopOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+ <span class="n">project_id</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span> <span class="n">zone</span><span class="o">=</span><span class="n">zone</span><span class="p">,</span> <span class="n">resource_id</span><span class="o">=</span><span class="n">resource_id</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="n">api_version</span><span class="o">=</span><span class="n">api_version</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+<div class="viewcode-block" id="GceInstanceStopOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcp_compute_operator.GceInstanceStopOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span><span class="o">.</span><span class="n">stop_instance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">zone</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">resource [...]
+
+
+<span class="n">SET_MACHINE_TYPE_VALIDATION_SPECIFICATION</span> <span class="o">=</span> <span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"machineType"</span><span class="p">,</span> <span class="n">regexp</span><span class="o">=</span><span class="s2">"^.+$"</span><span class="p">),</span>
+<span class="p">]</span>
+
+
+<div class="viewcode-block" id="GceSetMachineTypeOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcp_compute_operator.GceSetMachineTypeOperator">[docs]</a><span class="k">class</span> <span class="nc">GceSetMachineTypeOperator</span><span class="p">(</span><span class="n">GceBaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Changes the machine type for a stopped instance to the machine type specified in</span>
+<span class="sd"> the request.</span>
+
+<span class="sd"> :param project_id: Google Cloud Platform project where the Compute Engine</span>
+<span class="sd"> instance exists.</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :param zone: Google Cloud Platform zone where the instance exists.</span>
+<span class="sd"> :type zone: str</span>
+<span class="sd"> :param resource_id: Name of the Compute Engine instance resource.</span>
+<span class="sd"> :type resource_id: str</span>
+<span class="sd"> :param body: Body required by the Compute Engine setMachineType API, as described in</span>
+<span class="sd"> https://cloud.google.com/compute/docs/reference/rest/v1/instances/setMachineType#request-body</span>
+<span class="sd"> :type body: dict</span>
+<span class="sd"> :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: str</span>
+<span class="sd"> :param api_version: API version used (e.g. v1).</span>
+<span class="sd"> :type api_version: str</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'zone'</span><span class="p">,</span> <span class="s1">'resource_id'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span>
+ <span class="n">zone</span><span class="p">,</span>
+ <span class="n">resource_id</span><span class="p">,</span>
+ <span class="n">body</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="o">=</span><span class="s1">'v1'</span><span class="p">,</span>
+ <span class="n">validate_body</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">body</span> <span class="o">=</span> <span class="n">body</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_field_validator</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="k">if</span> <span class="n">validate_body</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_field_validator</span> <span class="o">=</span> <span class="n">GcpBodyFieldValidator</span><span class="p">(</span>
+ <span class="n">SET_MACHINE_TYPE_VALIDATION_SPECIFICATION</span><span class="p">,</span> <span class="n">api_version</span><span class="o">=</span><span class="n">api_version</span><span class="p">)</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">GceSetMachineTypeOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+ <span class="n">project_id</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span> <span class="n">zone</span><span class="o">=</span><span class="n">zone</span><span class="p">,</span> <span class="n">resource_id</span><span class="o">=</span><span class="n">resource_id</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="n">api_version</span><span class="o">=</span><span class="n">api_version</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_validate_all_body_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_field_validator</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_field_validator</span><span class="o">.</span><span class="n">validate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="p">)</span>
+
+<div class="viewcode-block" id="GceSetMachineTypeOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcp_compute_operator.GceSetMachineTypeOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_validate_all_body_fields</span><span class="p">()</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span><span class="o">.</span><span class="n">set_machine_type</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">zone</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">resource_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="p">)</span></div></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/operators/gcp_container_operator.html b/_modules/airflow/contrib/operators/gcp_container_operator.html
index 7b1235d..cb8b9f1 100644
--- a/_modules/airflow/contrib/operators/gcp_container_operator.html
+++ b/_modules/airflow/contrib/operators/gcp_container_operator.html
@@ -241,11 +241,11 @@
<span class="s1">'One of (project_id, name, location) is missing or incorrect'</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Operator has incorrect or missing input.'</span><span class="p">)</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="GKEClusterDeleteOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcp_container_operator.GKEClusterDeleteOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_check_input</span><span class="p">()</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">GKEClusterHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">)</span>
<span class="n">delete_result</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">delete_cluster</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">delete_result</span></div>
+ <span class="k">return</span> <span class="n">delete_result</span></div></div>
<div class="viewcode-block" id="GKEClusterCreateOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcp_container_operator.GKEClusterCreateOperator">[docs]</a><span class="k">class</span> <span class="nc">GKEClusterCreateOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
@@ -326,11 +326,11 @@
<span class="s1">'body[</span><span class="se">\'</span><span class="s1">initial_node_count</span><span class="se">\'</span><span class="s1">]) is missing or incorrect'</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Operator has incorrect or missing input.'</span><span class="p">)</span>
- <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+<div class="viewcode-block" id="GKEClusterCreateOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcp_container_operator.GKEClusterCreateOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_check_input</span><span class="p">()</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">GKEClusterHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">)</span>
<span class="n">create_op</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">create_cluster</span><span class="p">(</span><span class="n">cluster</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">create_op</span></div>
+ <span class="k">return</span> <span class="n">create_op</span></div></div>
</pre></div>
</div>
@@ -361,20 +361,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
diff --git a/_modules/airflow/contrib/operators/gcp_function_operator.html b/_modules/airflow/contrib/operators/gcp_function_operator.html
new file mode 100644
index 0000000..b84716a
--- /dev/null
+++ b/_modules/airflow/contrib/operators/gcp_function_operator.html
@@ -0,0 +1,522 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.gcp_function_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.gcp_function_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.gcp_function_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">import</span> <span class="nn">re</span>
+
+<span class="kn">from</span> <span class="nn">googleapiclient.errors</span> <span class="k">import</span> <span class="n">HttpError</span>
+
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.utils.gcp_field_validator</span> <span class="k">import</span> <span class="n">GcpBodyFieldValidator</span><span class="p">,</span> \
+ <span class="n">GcpFieldValidationException</span>
+<span class="kn">from</span> <span class="nn">airflow.version</span> <span class="k">import</span> <span class="n">version</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_function_hook</span> <span class="k">import</span> <span class="n">GcfHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<span class="k">def</span> <span class="nf">_validate_available_memory_in_mb</span><span class="p">(</span><span class="n">value</span><span class="p">):</span>
+ <span class="k">if</span> <span class="nb">int</span><span class="p">(</span><span class="n">value</span><span class="p">)</span> <span class="o"><=</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">GcpFieldValidationException</span><span class="p">(</span><span class="s2">"The available memory has to be greater than 0"</span><span class="p">)</span>
+
+
+<span class="k">def</span> <span class="nf">_validate_max_instances</span><span class="p">(</span><span class="n">value</span><span class="p">):</span>
+ <span class="k">if</span> <span class="nb">int</span><span class="p">(</span><span class="n">value</span><span class="p">)</span> <span class="o"><=</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">GcpFieldValidationException</span><span class="p">(</span>
+ <span class="s2">"The max instances parameter has to be greater than 0"</span><span class="p">)</span>
+
+
+<span class="n">CLOUD_FUNCTION_VALIDATION</span> <span class="o">=</span> <span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"name"</span><span class="p">,</span> <span class="n">regexp</span><span class="o">=</span><span class="s2">"^.+$"</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"description"</span><span class="p">,</span> <span class="n">regexp</span><span class="o">=</span><span class="s2">"^.+$"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"entryPoint"</span><span class="p">,</span> <span class="n">regexp</span><span class="o">=</span><span class="sa">r</span><span class="s1">'^.+$'</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"runtime"</span><span class="p">,</span> <span class="n">regexp</span><span class="o">=</span><span class="sa">r</span><span class="s1">'^.+$'</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"timeout"</span><span class="p">,</span> <span class="n">regexp</span><span class="o">=</span><span class="sa">r</span><span class="s1">'^.+$'</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"availableMemoryMb"</span><span class="p">,</span> <span class="n">custom_validation</span><span class="o">=</span><span class="n">_validate_available_memory_in_mb</span><span class="p">,</span>
+ <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"labels"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"environmentVariables"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"network"</span><span class="p">,</span> <span class="n">regexp</span><span class="o">=</span><span class="sa">r</span><span class="s1">'^.+$'</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"maxInstances"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">custom_validation</span><span class="o">=</span><span class="n">_validate_max_instances</span><span class="p">),</span>
+
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"source_code"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"union"</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"sourceArchiveUrl"</span><span class="p">,</span> <span class="n">regexp</span><span class="o">=</span><span class="sa">r</span><span class="s1">'^.+$'</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"sourceRepositoryUrl"</span><span class="p">,</span> <span class="n">regexp</span><span class="o">=</span><span class="sa">r</span><span class="s1">'^.+$'</span><span class="p">,</span> <span class="n">api_version</span><span class="o">=</span><span class="s1">'v1beta2'</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"sourceRepository"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"dict"</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"url"</span><span class="p">,</span> <span class="n">regexp</span><span class="o">=</span><span class="sa">r</span><span class="s1">'^.+$'</span><span class="p">)</span>
+ <span class="p">]),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"sourceUploadUrl"</span><span class="p">)</span>
+ <span class="p">]),</span>
+
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"trigger"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"union"</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"httpsTrigger"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"dict"</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">[</span>
+ <span class="c1"># This dict should be empty at input (url is added at output)</span>
+ <span class="p">]),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"eventTrigger"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"dict"</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"eventType"</span><span class="p">,</span> <span class="n">regexp</span><span class="o">=</span><span class="sa">r</span><span class="s1">'^.+$'</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"resource"</span><span class="p">,</span> <span class="n">regexp</span><span class="o">=</span><span class="sa">r</span><span class="s1">'^.+$'</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"service"</span><span class="p">,</span> <span class="n">regexp</span><span class="o">=</span><span class="sa">r</span><span class="s1">'^.+$'</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"failurePolicy"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"dict"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"retry"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"dict"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="p">])</span>
+ <span class="p">])</span>
+ <span class="p">]),</span>
+<span class="p">]</span>
+
+
+<div class="viewcode-block" id="GcfFunctionDeployOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcp_function_operator.GcfFunctionDeployOperator">[docs]</a><span class="k">class</span> <span class="nc">GcfFunctionDeployOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Creates a function in Google Cloud Functions.</span>
+
+<span class="sd"> :param project_id: Google Cloud Platform Project ID where the function should</span>
+<span class="sd"> be created.</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :param location: Google Cloud Platform region where the function should be created.</span>
+<span class="sd"> :type location: str</span>
+<span class="sd"> :param body: Body of the Cloud Functions definition. The body must be a</span>
+<span class="sd"> Cloud Functions dictionary as described in:</span>
+<span class="sd"> https://cloud.google.com/functions/docs/reference/rest/v1/projects.locations.functions</span>
+<span class="sd"> . Different API versions require different variants of the Cloud Functions</span>
+<span class="sd"> dictionary.</span>
+<span class="sd"> :type body: dict or google.cloud.functions.v1.CloudFunction</span>
+<span class="sd"> :param gcp_conn_id: The connection ID to use to connect to Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: str</span>
+<span class="sd"> :param api_version: API version used (for example v1 or v1beta1).</span>
+<span class="sd"> :type api_version: str</span>
+<span class="sd"> :param zip_path: Path to zip file containing source code of the function. If the path</span>
+<span class="sd"> is set, the sourceUploadUrl should not be specified in the body or it should</span>
+<span class="sd"> be empty. Then the zip file will be uploaded using the upload URL generated</span>
+<span class="sd"> via generateUploadUrl from the Cloud Functions API.</span>
+<span class="sd"> :type zip_path: str</span>
+<span class="sd"> :param validate_body: If set to False, body validation is not performed.</span>
+<span class="sd"> :type validate_body: bool</span>
+<span class="sd"> """</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span>
+ <span class="n">location</span><span class="p">,</span>
+ <span class="n">body</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="o">=</span><span class="s1">'v1'</span><span class="p">,</span>
+ <span class="n">zip_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">validate_body</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">location</span> <span class="o">=</span> <span class="n">location</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">full_location</span> <span class="o">=</span> <span class="s1">'projects/</span><span class="si">{}</span><span class="s1">/locations/</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">body</span> <span class="o">=</span> <span class="n">body</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">api_version</span> <span class="o">=</span> <span class="n">api_version</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">zip_path</span> <span class="o">=</span> <span class="n">zip_path</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">zip_path_preprocessor</span> <span class="o">=</span> <span class="n">ZipPathPreprocessor</span><span class="p">(</span><span class="n">body</span><span class="p">,</span> <span class="n">zip_path</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_field_validator</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="k">if</span> <span class="n">validate_body</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_field_validator</span> <span class="o">=</span> <span class="n">GcpBodyFieldValidator</span><span class="p">(</span><span class="n">CLOUD_FUNCTION_VALIDATION</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="o">=</span><span class="n">api_version</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span> <span class="o">=</span> <span class="n">GcfHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="n">api_version</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">api_version</span><span class [...]
+ <span class="bp">self</span><span class="o">.</span><span class="n">_validate_inputs</span><span class="p">()</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">GcfFunctionDeployOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_validate_inputs</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"The required parameter 'project_id' is missing"</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"The required parameter 'location' is missing"</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"The required parameter 'body' is missing"</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">zip_path_preprocessor</span><span class="o">.</span><span class="n">preprocess_body</span><span class="p">()</span>
+
+ <span class="k">def</span> <span class="nf">_validate_all_body_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_field_validator</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_field_validator</span><span class="o">.</span><span class="n">validate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_create_new_function</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span><span class="o">.</span><span class="n">create_new_function</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">full_location</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_update_function</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span><span class="o">.</span><span class="n">update_function</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="p">[</span><span class="s1">'name'</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</ [...]
+
+ <span class="k">def</span> <span class="nf">_check_if_function_exists</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="n">name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'name'</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">name</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">GcpFieldValidationException</span><span class="p">(</span><span class="s2">"The 'name' field should be present in "</span>
+ <span class="s2">"body: '</span><span class="si">{}</span><span class="s2">'."</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="p">))</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span><span class="o">.</span><span class="n">get_function</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
+ <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="n">status</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span>
+ <span class="k">if</span> <span class="n">status</span> <span class="o">==</span> <span class="mi">404</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">False</span>
+ <span class="k">raise</span> <span class="n">e</span>
+ <span class="k">return</span> <span class="kc">True</span>
+
+ <span class="k">def</span> <span class="nf">_upload_source_code</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span><span class="o">.</span><span class="n">upload_function_zip</span><span class="p">(</span><span class="n">parent</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">full_location</span><span class="p">,</span>
+ <span class="n">zip_path</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">zip_path</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_set_airflow_version_label</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="s1">'labels'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="p">[</span><span class="s1">'labels'</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="p">[</span><span class="s1">'labels'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
+ <span class="p">{</span><span class="s1">'airflow-version'</span><span class="p">:</span> <span class="s1">'v'</span> <span class="o">+</span> <span class="n">version</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'.'</span><span class="p">,</span> <span class="s1">'-'</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1" [...]
+
+<div class="viewcode-block" id="GcfFunctionDeployOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcp_function_operator.GcfFunctionDeployOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">zip_path_preprocessor</span><span class="o">.</span><span class="n">should_upload_function</span><span class="p">():</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="p">[</span><span class="n">SOURCE_UPLOAD_URL</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_upload_source_code</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_validate_all_body_fields</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_set_airflow_version_label</span><span class="p">()</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_check_if_function_exists</span><span class="p">():</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_create_new_function</span><span class="p">()</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_update_function</span><span class="p">()</span></div></div>
+
+
+<span class="n">SOURCE_ARCHIVE_URL</span> <span class="o">=</span> <span class="s1">'sourceArchiveUrl'</span>
+<span class="n">SOURCE_UPLOAD_URL</span> <span class="o">=</span> <span class="s1">'sourceUploadUrl'</span>
+<span class="n">SOURCE_REPOSITORY</span> <span class="o">=</span> <span class="s1">'sourceRepository'</span>
+<span class="n">ZIP_PATH</span> <span class="o">=</span> <span class="s1">'zip_path'</span>
+
+
+<span class="k">class</span> <span class="nc">ZipPathPreprocessor</span><span class="p">:</span>
+ <span class="sd">"""</span>
+<span class="sd"> Pre-processes zip path parameter.</span>
+
+<span class="sd"> Responsible for checking if the zip path parameter is correctly specified in</span>
+<span class="sd"> relation with source_code body fields. Non empty zip path parameter is special because</span>
+<span class="sd"> it is mutually exclusive with sourceArchiveUrl and sourceRepository body fields.</span>
+<span class="sd"> It is also mutually exclusive with non-empty sourceUploadUrl.</span>
+<span class="sd"> The pre-process modifies sourceUploadUrl body field in special way when zip_path</span>
+<span class="sd"> is not empty. An extra step is run when execute method is called and sourceUploadUrl</span>
+<span class="sd"> field value is set in the body with the value returned by generateUploadUrl Cloud</span>
+<span class="sd"> Function API method.</span>
+
+<span class="sd"> :param body: Body passed to the create/update method calls.</span>
+<span class="sd"> :type body: dict</span>
+<span class="sd"> :param zip_path: path to the zip file containing source code.</span>
+<span class="sd"> :type body: dict</span>
+
+<span class="sd"> """</span>
+ <span class="n">upload_function</span> <span class="o">=</span> <span class="kc">None</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">body</span><span class="p">,</span> <span class="n">zip_path</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">body</span> <span class="o">=</span> <span class="n">body</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">zip_path</span> <span class="o">=</span> <span class="n">zip_path</span>
+
+ <span class="nd">@staticmethod</span>
+ <span class="k">def</span> <span class="nf">_is_present_and_empty</span><span class="p">(</span><span class="n">dictionary</span><span class="p">,</span> <span class="n">field</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">field</span> <span class="ow">in</span> <span class="n">dictionary</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">dictionary</span><span class="p">[</span><span class="n">field</span><span class="p">]</span>
+
+ <span class="k">def</span> <span class="nf">_verify_upload_url_and_no_zip_path</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_is_present_and_empty</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="p">,</span> <span class="n">SOURCE_UPLOAD_URL</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">zip_path</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s2">"Parameter '</span><span class="si">{}</span><span class="s2">' is empty in the body and argument '</span><span class="si">{}</span><span class="s2">' "</span>
+ <span class="s2">"is missing or empty. You need to have non empty '</span><span class="si">{}</span><span class="s2">' "</span>
+ <span class="s2">"when '</span><span class="si">{}</span><span class="s2">' is present and empty."</span><span class="o">.</span>
+ <span class="nb">format</span><span class="p">(</span><span class="n">SOURCE_UPLOAD_URL</span><span class="p">,</span> <span class="n">ZIP_PATH</span><span class="p">,</span> <span class="n">ZIP_PATH</span><span class="p">,</span> <span class="n">SOURCE_UPLOAD_URL</span><span class="p">))</span>
+
+ <span class="k">def</span> <span class="nf">_verify_upload_url_and_zip_path</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">SOURCE_UPLOAD_URL</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">body</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">zip_path</span><span class="p">:</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="p">[</span><span class="n">SOURCE_UPLOAD_URL</span><span class="p">]:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">upload_function</span> <span class="o">=</span> <span class="kc">True</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Only one of '</span><span class="si">{}</span><span class="s2">' in body or '</span><span class="si">{}</span><span class="s2">' argument "</span>
+ <span class="s2">"allowed. Found both."</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">SOURCE_UPLOAD_URL</span><span class="p">,</span> <span class="n">ZIP_PATH</span><span class="p">))</span>
+
+ <span class="k">def</span> <span class="nf">_verify_archive_url_and_zip_path</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">SOURCE_ARCHIVE_URL</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">body</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">zip_path</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Only one of '</span><span class="si">{}</span><span class="s2">' in body or '</span><span class="si">{}</span><span class="s2">' argument "</span>
+ <span class="s2">"allowed. Found both."</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">SOURCE_ARCHIVE_URL</span><span class="p">,</span> <span class="n">ZIP_PATH</span><span class="p">))</span>
+
+ <span class="k">def</span> <span class="nf">should_upload_function</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">upload_function</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'validate() method has to be invoked before '</span>
+ <span class="s1">'should_upload_function'</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">upload_function</span>
+
+ <span class="k">def</span> <span class="nf">preprocess_body</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_verify_archive_url_and_zip_path</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_verify_upload_url_and_zip_path</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_verify_upload_url_and_no_zip_path</span><span class="p">()</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">upload_function</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">upload_function</span> <span class="o">=</span> <span class="kc">False</span>
+
+
+<span class="n">FUNCTION_NAME_PATTERN</span> <span class="o">=</span> <span class="s1">'^projects/[^/]+/locations/[^/]+/functions/[^/]+$'</span>
+<span class="n">FUNCTION_NAME_COMPILED_PATTERN</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="n">FUNCTION_NAME_PATTERN</span><span class="p">)</span>
+
+
+<div class="viewcode-block" id="GcfFunctionDeleteOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcp_function_operator.GcfFunctionDeleteOperator">[docs]</a><span class="k">class</span> <span class="nc">GcfFunctionDeleteOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Deletes the specified function from Google Cloud Functions.</span>
+
+<span class="sd"> :param name: A fully-qualified function name, matching</span>
+<span class="sd"> the pattern: `^projects/[^/]+/locations/[^/]+/functions/[^/]+$`</span>
+<span class="sd"> :type name: str</span>
+<span class="sd"> :param gcp_conn_id: The connection ID to use to connect to Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: str</span>
+<span class="sd"> :param api_version: API version used (for example v1 or v1beta1).</span>
+<span class="sd"> :type api_version: str</span>
+<span class="sd"> """</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">name</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="o">=</span><span class="s1">'v1'</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">api_version</span> <span class="o">=</span> <span class="n">api_version</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_validate_inputs</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">GcfHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="n">api_version</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">api_version</span><span class= [...]
+ <span class="nb">super</span><span class="p">(</span><span class="n">GcfFunctionDeleteOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_validate_inputs</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span><span class="s1">'Empty parameter: name'</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">pattern</span> <span class="o">=</span> <span class="n">FUNCTION_NAME_COMPILED_PATTERN</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">pattern</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span>
+ <span class="s1">'Parameter name must match pattern: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">FUNCTION_NAME_PATTERN</span><span class="p">))</span>
+
+<div class="viewcode-block" id="GcfFunctionDeleteOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcp_function_operator.GcfFunctionDeleteOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">delete_function</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
+ <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="n">status</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span>
+ <span class="k">if</span> <span class="n">status</span> <span class="o">==</span> <span class="mi">404</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'The function does not exist in this project'</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">'An error occurred. Exiting.'</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="n">e</span></div></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_modules/airflow/contrib/operators/gcp_sql_operator.html b/_modules/airflow/contrib/operators/gcp_sql_operator.html
new file mode 100644
index 0000000..62c01b8
--- /dev/null
+++ b/_modules/airflow/contrib/operators/gcp_sql_operator.html
@@ -0,0 +1,737 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.operators.gcp_sql_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.operators.gcp_sql_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.operators.gcp_sql_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">from</span> <span class="nn">googleapiclient.errors</span> <span class="k">import</span> <span class="n">HttpError</span>
+
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_sql_hook</span> <span class="k">import</span> <span class="n">CloudSqlHook</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.utils.gcp_field_validator</span> <span class="k">import</span> <span class="n">GcpBodyFieldValidator</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+<span class="n">SETTINGS</span> <span class="o">=</span> <span class="s1">'settings'</span>
+<span class="n">SETTINGS_VERSION</span> <span class="o">=</span> <span class="s1">'settingsVersion'</span>
+
+<span class="n">CLOUD_SQL_VALIDATION</span> <span class="o">=</span> <span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"name"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"settings"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"dict"</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"tier"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"backupConfiguration"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"dict"</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"binaryLogEnabled"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"enabled"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"replicationLogArchivingEnabled"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"startTime"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="p">],</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"activationPolicy"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"authorizedGaeApplications"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"list"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"crashSafeReplicationEnabled"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"dataDiskSizeGb"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"dataDiskType"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"databaseFlags"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"list"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"ipConfiguration"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"dict"</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"authorizedNetworks"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"list"</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"expirationTime"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"name"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"value"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="p">],</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"ipv4Enabled"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"privateNetwork"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"requireSsl"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="p">],</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"locationPreference"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"dict"</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"followGaeApplication"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"zone"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="p">],</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"maintenanceWindow"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"dict"</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"hour"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"day"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"updateTrack"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="p">],</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"pricingPlan"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"replicationType"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"storageAutoResize"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"storageAutoResizeLimit"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"userLabels"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"dict"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="p">]),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"databaseVersion"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"failoverReplica"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"dict"</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"name"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
+ <span class="p">],</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"masterInstanceName"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"onPremisesConfiguration"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"dict"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"region"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"replicaConfiguration"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"dict"</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"failoverTarget"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"mysqlReplicaConfiguration"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"dict"</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"caCertificate"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"clientCertificate"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"clientKey"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"connectRetryInterval"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"dumpFilePath"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"masterHeartbeatPeriod"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"password"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"sslCipher"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"username"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"verifyServerCertificate"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="p">],</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="p">],</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+<span class="p">]</span>
+<span class="n">CLOUD_SQL_EXPORT_VALIDATION</span> <span class="o">=</span> <span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"exportContext"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"dict"</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"fileType"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"uri"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"databases"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"list"</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"sqlExportOptions"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"dict"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"tables"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"list"</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"schemaOnly"</span><span class="p">)</span>
+ <span class="p">]),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"csvExportOptions"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"dict"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"selectQuery"</span><span class="p">)</span>
+ <span class="p">])</span>
+ <span class="p">])</span>
+<span class="p">]</span>
+<span class="n">CLOUD_SQL_IMPORT_VALIDATION</span> <span class="o">=</span> <span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"importContext"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"dict"</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"fileType"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"uri"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"database"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"importUser"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"csvImportOptions"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"dict"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"table"</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"columns"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"list"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="p">])</span>
+ <span class="p">])</span>
+<span class="p">]</span>
+<span class="n">CLOUD_SQL_DATABASE_INSERT_VALIDATION</span> <span class="o">=</span> <span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"instance"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"name"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"project"</span><span class="p">,</span> <span class="n">allow_empty</span><span class="o">=</span><span class="kc">False</span><span class="p">),</span>
+<span class="p">]</span>
+<span class="n">CLOUD_SQL_DATABASE_PATCH_VALIDATION</span> <span class="o">=</span> <span class="p">[</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"instance"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"name"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"project"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"etag"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"charset"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+ <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"collation"</span><span class="p">,</span> <span class="n">optional</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
+<span class="p">]</span>
+
+
+<span class="k">class</span> <span class="nc">CloudSqlBaseOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Abstract base operator for Google Cloud SQL operators to inherit from.</span>
+
+<span class="sd"> :param project_id: Project ID of the Google Cloud Platform project to operate it.</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :param instance: Cloud SQL instance ID. This does not include the project ID.</span>
+<span class="sd"> :type instance: str</span>
+<span class="sd"> :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: str</span>
+<span class="sd"> :param api_version: API version used (e.g. v1).</span>
+<span class="sd"> :type api_version: str</span>
+<span class="sd"> """</span>
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span>
+ <span class="n">instance</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="o">=</span><span class="s1">'v1beta4'</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">instance</span> <span class="o">=</span> <span class="n">instance</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">api_version</span> <span class="o">=</span> <span class="n">api_version</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_validate_inputs</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span> <span class="o">=</span> <span class="n">CloudSqlHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">api_version</span><span class="p">)</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">CloudSqlBaseOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_validate_inputs</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"The required parameter 'project_id' is empty"</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">instance</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"The required parameter 'instance' is empty"</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_check_if_instance_exists</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">instance</span><span class="p">):</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span><span class="o">.</span><span class="n">get_instance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">instance</span><span class="p">)</span>
+ <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="n">status</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span>
+ <span class="k">if</span> <span class="n">status</span> <span class="o">==</span> <span class="mi">404</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">False</span>
+ <span class="k">raise</span> <span class="n">e</span>
+
+ <span class="k">def</span> <span class="nf">_check_if_db_exists</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">db_name</span><span class="p">):</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span><span class="o">.</span><span class="n">get_database</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">instance</span><span class="p">,</span> <span class="n">db_name</span><span class="p">)</span>
+ <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="n">status</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span>
+ <span class="k">if</span> <span class="n">status</span> <span class="o">==</span> <span class="mi">404</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">False</span>
+ <span class="k">raise</span> <span class="n">e</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="k">pass</span>
+
+ <span class="nd">@staticmethod</span>
+ <span class="k">def</span> <span class="nf">_get_settings_version</span><span class="p">(</span><span class="n">instance</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">instance</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">SETTINGS</span><span class="p">)</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">SETTINGS_VERSION</span><span class="p">)</span>
+
+
+<div class="viewcode-block" id="CloudSqlInstanceCreateOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceCreateOperator">[docs]</a><span class="k">class</span> <span class="nc">CloudSqlInstanceCreateOperator</span><span class="p">(</span><span class="n">CloudSqlBaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Creates a new Cloud SQL instance.</span>
+<span class="sd"> If an instance with the same name exists, no action will be taken and</span>
+<span class="sd"> the operator will succeed.</span>
+
+<span class="sd"> :param project_id: Project ID of the project to which the newly created Cloud SQL</span>
+<span class="sd"> instances should belong.</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :param body: Body required by the Cloud SQL insert API, as described in</span>
+<span class="sd"> https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/insert</span>
+<span class="sd"> #request-body</span>
+<span class="sd"> :type body: dict</span>
+<span class="sd"> :param instance: Cloud SQL instance ID. This does not include the project ID.</span>
+<span class="sd"> :type instance: str</span>
+<span class="sd"> :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: str</span>
+<span class="sd"> :param api_version: API version used (e.g. v1beta4).</span>
+<span class="sd"> :type api_version: str</span>
+<span class="sd"> :param validate_body: True if body should be validated, False otherwise.</span>
+<span class="sd"> :type validate_body: bool</span>
+<span class="sd"> """</span>
+ <span class="c1"># [START gcp_sql_create_template_fields]</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span>
+ <span class="c1"># [END gcp_sql_create_template_fields]</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span>
+ <span class="n">body</span><span class="p">,</span>
+ <span class="n">instance</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="o">=</span><span class="s1">'v1beta4'</span><span class="p">,</span>
+ <span class="n">validate_body</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">body</span> <span class="o">=</span> <span class="n">body</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">validate_body</span> <span class="o">=</span> <span class="n">validate_body</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">CloudSqlInstanceCreateOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+ <span class="n">project_id</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span> <span class="n">instance</span><span class="o">=</span><span class="n">instance</span><span class="p">,</span> <span class="n">gcp_conn_id</span><span class="o">=</span><span class="n">gcp_conn_id</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="o">=</span><span class="n">api_version</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_validate_inputs</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">CloudSqlInstanceCreateOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">_validate_inputs</span><span class="p">()</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"The required parameter 'body' is empty"</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_validate_body_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">validate_body</span><span class="p">:</span>
+ <span class="n">GcpBodyFieldValidator</span><span class="p">(</span><span class="n">CLOUD_SQL_VALIDATION</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">api_version</span><span class="p">)</span><span class="o">.</span><span class="n">validate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="p">)</span>
+
+<div class="viewcode-block" id="CloudSqlInstanceCreateOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceCreateOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_validate_body_fields</span><span class="p">()</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_check_if_instance_exists</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">instance</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span><span class="o">.</span><span class="n">create_instance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Cloud SQL instance with ID </span><span class="si">{}</span><span class="s2"> already exists. "</span>
+ <span class="s2">"Aborting create."</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">instance</span><span class="p">))</span>
+ <span class="k">return</span> <span class="kc">True</span></div></div>
+
+
+<div class="viewcode-block" id="CloudSqlInstancePatchOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstancePatchOperator">[docs]</a><span class="k">class</span> <span class="nc">CloudSqlInstancePatchOperator</span><span class="p">(</span><span class="n">CloudSqlBaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Updates settings of a Cloud SQL instance.</span>
+
+<span class="sd"> Caution: This is a partial update, so only included values for the settings will be</span>
+<span class="sd"> updated.</span>
+
+<span class="sd"> In the request body, supply the relevant portions of an instance resource, according</span>
+<span class="sd"> to the rules of patch semantics.</span>
+<span class="sd"> https://cloud.google.com/sql/docs/mysql/admin-api/how-tos/performance#patch</span>
+
+<span class="sd"> :param project_id: Project ID of the project that contains the instance.</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :param body: Body required by the Cloud SQL patch API, as described in</span>
+<span class="sd"> https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/patch#request-body</span>
+<span class="sd"> :type body: dict</span>
+<span class="sd"> :param instance: Cloud SQL instance ID. This does not include the project ID.</span>
+<span class="sd"> :type instance: str</span>
+<span class="sd"> :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: str</span>
+<span class="sd"> :param api_version: API version used (e.g. v1beta4).</span>
+<span class="sd"> :type api_version: str</span>
+<span class="sd"> """</span>
+ <span class="c1"># [START gcp_sql_patch_template_fields]</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span>
+ <span class="c1"># [END gcp_sql_patch_template_fields]</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span>
+ <span class="n">body</span><span class="p">,</span>
+ <span class="n">instance</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="o">=</span><span class="s1">'v1beta4'</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">body</span> <span class="o">=</span> <span class="n">body</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">CloudSqlInstancePatchOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+ <span class="n">project_id</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span> <span class="n">instance</span><span class="o">=</span><span class="n">instance</span><span class="p">,</span> <span class="n">gcp_conn_id</span><span class="o">=</span><span class="n">gcp_conn_id</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="o">=</span><span class="n">api_version</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_validate_inputs</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">CloudSqlInstancePatchOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">_validate_inputs</span><span class="p">()</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"The required parameter 'body' is empty"</span><span class="p">)</span>
+
+<div class="viewcode-block" id="CloudSqlInstancePatchOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstancePatchOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_check_if_instance_exists</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">instance</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Cloud SQL instance with ID </span><span class="si">{}</span><span class="s1"> does not exist. '</span>
+ <span class="s1">'Please specify another instance to patch.'</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">instance</span><span class="p">))</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span><span class="o">.</span><span class="n">patch_instance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ins [...]
+
+
+<div class="viewcode-block" id="CloudSqlInstanceDeleteOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDeleteOperator">[docs]</a><span class="k">class</span> <span class="nc">CloudSqlInstanceDeleteOperator</span><span class="p">(</span><span class="n">CloudSqlBaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Deletes a Cloud SQL instance.</span>
+
+<span class="sd"> :param project_id: Project ID of the project that contains the instance to be deleted.</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :param instance: Cloud SQL instance ID. This does not include the project ID.</span>
+<span class="sd"> :type instance: str</span>
+<span class="sd"> :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: str</span>
+<span class="sd"> :param api_version: API version used (e.g. v1beta4).</span>
+<span class="sd"> :type api_version: str</span>
+<span class="sd"> """</span>
+ <span class="c1"># [START gcp_sql_delete_template_fields]</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span>
+ <span class="c1"># [END gcp_sql_delete_template_fields]</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span>
+ <span class="n">instance</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="o">=</span><span class="s1">'v1beta4'</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">CloudSqlInstanceDeleteOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+ <span class="n">project_id</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span> <span class="n">instance</span><span class="o">=</span><span class="n">instance</span><span class="p">,</span> <span class="n">gcp_conn_id</span><span class="o">=</span><span class="n">gcp_conn_id</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="o">=</span><span class="n">api_version</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+<div class="viewcode-block" id="CloudSqlInstanceDeleteOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDeleteOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_check_if_instance_exists</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">instance</span><span class="p">):</span>
+ <span class="nb">print</span><span class="p">(</span><span class="s2">"Cloud SQL instance with ID </span><span class="si">{}</span><span class="s2"> does not exist. Aborting delete."</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">instance</span><span class="p">))</span>
+ <span class="k">return</span> <span class="kc">True</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span><span class="o">.</span><span class="n">delete_instance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">instance</span><span class="p">)</span></div></div>
+
+
+<div class="viewcode-block" id="CloudSqlInstanceDatabaseCreateOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabaseCreateOperator">[docs]</a><span class="k">class</span> <span class="nc">CloudSqlInstanceDatabaseCreateOperator</span><span class="p">(</span><span class="n">CloudSqlBaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Creates a new database inside a Cloud SQL instance.</span>
+
+<span class="sd"> :param project_id: Project ID of the project that contains the instance.</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :param instance: Database instance ID. This does not include the project ID.</span>
+<span class="sd"> :type instance: str</span>
+<span class="sd"> :param body: The request body, as described in</span>
+<span class="sd"> https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/databases/insert#request-body</span>
+<span class="sd"> :type body: dict</span>
+<span class="sd"> :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: str</span>
+<span class="sd"> :param api_version: API version used (e.g. v1beta4).</span>
+<span class="sd"> :type api_version: str</span>
+<span class="sd"> :param validate_body: Whether the body should be validated. Defaults to True.</span>
+<span class="sd"> :type validate_body: bool</span>
+<span class="sd"> """</span>
+ <span class="c1"># [START gcp_sql_db_create_template_fields]</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span>
+ <span class="c1"># [END gcp_sql_db_create_template_fields]</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span>
+ <span class="n">instance</span><span class="p">,</span>
+ <span class="n">body</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="o">=</span><span class="s1">'v1beta4'</span><span class="p">,</span>
+ <span class="n">validate_body</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">body</span> <span class="o">=</span> <span class="n">body</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">validate_body</span> <span class="o">=</span> <span class="n">validate_body</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">CloudSqlInstanceDatabaseCreateOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+ <span class="n">project_id</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span> <span class="n">instance</span><span class="o">=</span><span class="n">instance</span><span class="p">,</span> <span class="n">gcp_conn_id</span><span class="o">=</span><span class="n">gcp_conn_id</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="o">=</span><span class="n">api_version</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_validate_inputs</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">CloudSqlInstanceDatabaseCreateOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">_validate_inputs</span><span class="p">()</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"The required parameter 'body' is empty"</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_validate_body_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">validate_body</span><span class="p">:</span>
+ <span class="n">GcpBodyFieldValidator</span><span class="p">(</span><span class="n">CLOUD_SQL_DATABASE_INSERT_VALIDATION</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">api_version</span><span class="p">)</span><span class="o">.</span><span class="n">validate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="p">)</span>
+
+<div class="viewcode-block" id="CloudSqlInstanceDatabaseCreateOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabaseCreateOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_validate_body_fields</span><span class="p">()</span>
+ <span class="n">database</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"name"</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">database</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Body doesn't contain 'name'. Cannot check if the"</span>
+ <span class="s2">" database already exists in the instance </span><span class="si">{}</span><span class="s2">."</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">instance</span><span class="p">))</span>
+ <span class="k">return</span> <span class="kc">False</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_check_if_db_exists</span><span class="p">(</span><span class="n">database</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Cloud SQL instance with ID </span><span class="si">{}</span><span class="s2"> already contains database"</span>
+ <span class="s2">" '</span><span class="si">{}</span><span class="s2">'. Aborting database insert."</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">instance</span><span class="p">,</span> <span class="n">database</span><span class="p">))</span>
+ <span class="k">return</span> <span class="kc">True</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span><span class="o">.</span><span class="n">create_database</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">instance</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n [...]
+
+
+<div class="viewcode-block" id="CloudSqlInstanceDatabasePatchOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabasePatchOperator">[docs]</a><span class="k">class</span> <span class="nc">CloudSqlInstanceDatabasePatchOperator</span><span class="p">(</span><span class="n">CloudSqlBaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Updates a resource containing information about a database inside a Cloud SQL</span>
+<span class="sd"> instance using patch semantics.</span>
+<span class="sd"> See: https://cloud.google.com/sql/docs/mysql/admin-api/how-tos/performance#patch</span>
+
+<span class="sd"> :param project_id: Project ID of the project that contains the instance.</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :param instance: Database instance ID. This does not include the project ID.</span>
+<span class="sd"> :type instance: str</span>
+<span class="sd"> :param database: Name of the database to be updated in the instance.</span>
+<span class="sd"> :type database: str</span>
+<span class="sd"> :param body: The request body, as described in</span>
+<span class="sd"> https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/databases/patch#request-body</span>
+<span class="sd"> :type body: dict</span>
+<span class="sd"> :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: str</span>
+<span class="sd"> :param api_version: API version used (e.g. v1beta4).</span>
+<span class="sd"> :type api_version: str</span>
+<span class="sd"> :param validate_body: Whether the body should be validated. Defaults to True.</span>
+<span class="sd"> :type validate_body: bool</span>
+<span class="sd"> """</span>
+ <span class="c1"># [START gcp_sql_db_patch_template_fields]</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'database'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span>
+ <span class="s1">'api_version'</span><span class="p">)</span>
+ <span class="c1"># [END gcp_sql_db_patch_template_fields]</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span>
+ <span class="n">instance</span><span class="p">,</span>
+ <span class="n">database</span><span class="p">,</span>
+ <span class="n">body</span><span class="p">,</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_default'</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="o">=</span><span class="s1">'v1beta4'</span><span class="p">,</span>
+ <span class="n">validate_body</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">database</span> <span class="o">=</span> <span class="n">database</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">body</span> <span class="o">=</span> <span class="n">body</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">validate_body</span> <span class="o">=</span> <span class="n">validate_body</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">CloudSqlInstanceDatabasePatchOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+ <span class="n">project_id</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span> <span class="n">instance</span><span class="o">=</span><span class="n">instance</span><span class="p">,</span> <span class="n">gcp_conn_id</span><span class="o">=</span><span class="n">gcp_conn_id</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="o">=</span><span class="n">api_version</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_validate_inputs</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">CloudSqlInstanceDatabasePatchOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">_validate_inputs</span><span class="p">()</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"The required parameter 'body' is empty"</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">database</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"The required parameter 'database' is empty"</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_validate_body_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">validate_body</span><span class="p">:</span>
+ <span class="n">GcpBodyFieldValidator</span><span class="p">(</span><span class="n">CLOUD_SQL_DATABASE_PATCH_VALIDATION</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">api_version</span><span class="p">)</span><span class="o">.</span><span class="n">validate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="p">)</span>
+
+<div class="viewcode-block" id="CloudSqlInstanceDatabasePatchOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabasePatchOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_validate_body_fields</span><span class="p">()</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_check_if_db_exists</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">database</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Cloud SQL instance with ID </span><span class="si">{}</span><span class="s2"> does not contain "</span>
+ <span class="s2">"database '</span><span class="si">{}</span><span class="s2">'. "</span>
+ <span class="s2">"Please specify another database to patch."</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">instance</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">database</span><span class="p">))</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_hook</span><span class="o">.</span><span class="n">patch_database</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">instance</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">database</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">body</span><span class="p">)</span></div></div>
+
+
+<div class="viewcode-block" id="CloudSqlInstanceDatabaseDeleteOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabaseDeleteOperator">[docs]</a><span class="k">class</span> <span class="nc">CloudSqlInstanceDatabaseDeleteOperator</span><span class="p">(</span><span class="n">CloudSqlBaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Deletes a database from a Cloud SQL instance.</span>
+
+<span class="sd"> :param project_id: Project ID of the project that contains the instance.</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :param instance: Database instance ID. This does not include the project ID.</span>
+<span class="sd"> :type instance: str</span>
+<span class="sd"> :param database: Name of the database to be deleted in the instance.</span>
+<span class="sd"> :type database: str</span>
+<span class="sd"> :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.</span>
+<span class="sd"> :type gcp_conn_id: str</span>
+<span class="sd"> :param api_version: API version used (e.g. v1beta4).</span>
+<span class="sd"> :type api_version: str</span>
+<span class="sd"> """</span>
+ <span class="c1"># [START gcp_sql_db_delete_template_fields]</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'database'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span>
+ <span class="s1">'api_version'</span><span class="p">)</span>
+ <span class="c1"># [END gcp_sql_db_delete_template_fields]</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span>
... 26255 lines suppressed ...