You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by as...@apache.org on 2019/11/27 19:10:28 UTC
[airflow-site] 15/30: Add separate folder for 1.10.2
This is an automated email from the ASF dual-hosted git repository.
ash pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/airflow-site.git
commit 42910ec5d11bb8a26858920865f8bf23364d6280
Author: Kaxil Naik <ka...@gmail.com>
AuthorDate: Wed Jan 23 00:50:14 2019 +0000
Add separate folder for 1.10.2
---
1.10.2/.buildinfo | 4 +
1.10.2/_images/adhoc.png | Bin 0 -> 182773 bytes
1.10.2/_images/airflow.gif | Bin 0 -> 622832 bytes
1.10.2/_images/apache.jpg | Bin 0 -> 43364 bytes
1.10.2/_images/branch_bad.png | Bin 0 -> 8825 bytes
1.10.2/_images/branch_good.png | Bin 0 -> 12035 bytes
1.10.2/_images/chart.png | Bin 0 -> 169382 bytes
1.10.2/_images/chart_form.png | Bin 0 -> 203224 bytes
1.10.2/_images/code.png | Bin 0 -> 285906 bytes
1.10.2/_images/connection_create.png | Bin 0 -> 41547 bytes
1.10.2/_images/connection_edit.png | Bin 0 -> 53636 bytes
1.10.2/_images/connections.png | Bin 0 -> 48442 bytes
1.10.2/_images/context.png | Bin 0 -> 149588 bytes
1.10.2/_images/dags.png | Bin 0 -> 217069 bytes
1.10.2/_images/duration.png | Bin 0 -> 250846 bytes
1.10.2/_images/gantt.png | Bin 0 -> 121795 bytes
1.10.2/_images/graph.png | Bin 0 -> 152205 bytes
1.10.2/_images/latest_only_with_trigger.png | Bin 0 -> 40034 bytes
1.10.2/_images/pin_large.png | Bin 0 -> 358276 bytes
1.10.2/_images/subdag_after.png | Bin 0 -> 30245 bytes
1.10.2/_images/subdag_before.png | Bin 0 -> 70382 bytes
1.10.2/_images/subdag_zoom.png | Bin 0 -> 150185 bytes
1.10.2/_images/tree.png | Bin 0 -> 128024 bytes
1.10.2/_images/variable_hidden.png | Bin 0 -> 154299 bytes
.../contrib/executors/kubernetes_executor.html | 897 ++
.../airflow/contrib/executors/mesos_executor.html | 549 +
.../airflow/contrib/hooks/aws_athena_hook.html | 369 +
.../airflow/contrib/hooks/aws_dynamodb_hook.html | 288 +
.../airflow/contrib/hooks/aws_firehose_hook.html | 275 +
.../contrib/hooks/aws_glue_catalog_hook.html | 337 +
.../_modules/airflow/contrib/hooks/aws_hook.html | 424 +
.../airflow/contrib/hooks/aws_lambda_hook.html | 287 +
.../airflow/contrib/hooks/azure_cosmos_hook.html | 506 +
.../contrib/hooks/azure_data_lake_hook.html | 360 +
.../contrib/hooks/azure_fileshare_hook.html | 431 +
.../airflow/contrib/hooks/bigquery_hook.html | 2184 +++
.../airflow/contrib/hooks/cassandra_hook.html | 416 +
.../airflow/contrib/hooks/cloudant_hook.html | 302 +
.../airflow/contrib/hooks/databricks_hook.html | 495 +
.../airflow/contrib/hooks/datadog_hook.html | 361 +
.../airflow/contrib/hooks/datastore_hook.html | 425 +
.../contrib/hooks/discord_webhook_hook.html | 359 +
.../_modules/airflow/contrib/hooks/emr_hook.html | 276 +
1.10.2/_modules/airflow/contrib/hooks/fs_hook.html | 265 +
.../_modules/airflow/contrib/hooks/ftp_hook.html | 534 +
.../airflow/contrib/hooks/gcp_api_base_hook.html | 414 +
.../airflow/contrib/hooks/gcp_compute_hook.html | 554 +
.../airflow/contrib/hooks/gcp_dataflow_hook.html | 513 +
.../airflow/contrib/hooks/gcp_dataproc_hook.html | 464 +
.../airflow/contrib/hooks/gcp_function_hook.html | 417 +
.../airflow/contrib/hooks/gcp_mlengine_hook.html | 490 +
.../airflow/contrib/hooks/gcp_pubsub_hook.html | 504 +
.../airflow/contrib/hooks/gcp_sql_hook.html | 1216 ++
.../airflow/contrib/hooks/gcp_transfer_hook.html | 326 +
.../_modules/airflow/contrib/hooks/gcs_hook.html | 895 ++
.../_modules/airflow/contrib/hooks/imap_hook.html | 499 +
.../_modules/airflow/contrib/hooks/jira_hook.html | 306 +
.../_modules/airflow/contrib/hooks/mongo_hook.html | 356 +
.../airflow/contrib/hooks/openfaas_hook.html | 318 +
.../_modules/airflow/contrib/hooks/pinot_hook.html | 324 +
.../airflow/contrib/hooks/qubole_hook.html | 433 +
.../_modules/airflow/contrib/hooks/redis_hook.html | 312 +
.../airflow/contrib/hooks/redshift_hook.html | 332 +
.../airflow/contrib/hooks/sagemaker_hook.html | 981 ++
.../airflow/contrib/hooks/salesforce_hook.html | 544 +
.../_modules/airflow/contrib/hooks/sftp_hook.html | 433 +
.../airflow/contrib/hooks/slack_webhook_hook.html | 348 +
.../airflow/contrib/hooks/snowflake_hook.html | 317 +
.../airflow/contrib/hooks/spark_jdbc_hook.html | 465 +
.../airflow/contrib/hooks/spark_sql_hook.html | 381 +
.../airflow/contrib/hooks/spark_submit_hook.html | 786 +
.../_modules/airflow/contrib/hooks/sqoop_hook.html | 571 +
.../_modules/airflow/contrib/hooks/ssh_hook.html | 468 +
.../airflow/contrib/hooks/vertica_hook.html | 272 +
.../_modules/airflow/contrib/hooks/wasb_hook.html | 410 +
.../_modules/airflow/contrib/hooks/winrm_hook.html | 356 +
.../airflow/contrib/kubernetes/secret.html | 260 +
.../contrib/operators/adls_list_operator.html | 289 +
.../airflow/contrib/operators/adls_to_gcs.html | 365 +
.../contrib/operators/aws_athena_operator.html | 317 +
.../contrib/operators/awsbatch_operator.html | 398 +
.../contrib/operators/azure_cosmos_operator.html | 288 +
.../contrib/operators/bigquery_check_operator.html | 364 +
.../contrib/operators/bigquery_get_data.html | 335 +
.../contrib/operators/bigquery_operator.html | 856 ++
.../operators/bigquery_table_delete_operator.html | 285 +
.../contrib/operators/bigquery_to_bigquery.html | 314 +
.../airflow/contrib/operators/bigquery_to_gcs.html | 324 +
.../contrib/operators/cassandra_to_gcs.html | 574 +
.../contrib/operators/databricks_operator.html | 699 +
.../contrib/operators/dataflow_operator.html | 593 +
.../contrib/operators/dataproc_operator.html | 1678 ++
.../operators/datastore_export_operator.html | 329 +
.../operators/datastore_import_operator.html | 317 +
.../operators/discord_webhook_operator.html | 317 +
.../airflow/contrib/operators/druid_operator.html | 283 +
.../airflow/contrib/operators/ecs_operator.html | 371 +
.../contrib/operators/emr_add_steps_operator.html | 282 +
.../operators/emr_create_job_flow_operator.html | 293 +
.../operators/emr_terminate_job_flow_operator.html | 276 +
.../airflow/contrib/operators/file_to_gcs.html | 301 +
.../airflow/contrib/operators/file_to_wasb.html | 283 +
.../contrib/operators/gcp_compute_operator.html | 681 +
.../contrib/operators/gcp_function_operator.html | 539 +
.../contrib/operators/gcp_sql_operator.html | 988 ++
.../contrib/operators/gcs_acl_operator.html | 345 +
.../contrib/operators/gcs_download_operator.html | 310 +
.../contrib/operators/gcs_list_operator.html | 311 +
.../airflow/contrib/operators/gcs_operator.html | 341 +
.../airflow/contrib/operators/gcs_to_bq.html | 488 +
.../airflow/contrib/operators/gcs_to_gcs.html | 387 +
.../operators/gcs_to_gcs_transfer_operator.html | 346 +
.../airflow/contrib/operators/gcs_to_s3.html | 344 +
.../contrib/operators/hipchat_operator.html | 354 +
.../contrib/operators/hive_to_dynamodb.html | 328 +
.../airflow/contrib/operators/jira_operator.html | 313 +
.../contrib/operators/kubernetes_pod_operator.html | 422 +
.../contrib/operators/mlengine_operator.html | 829 +
.../airflow/contrib/operators/mongo_to_s3.html | 347 +
.../airflow/contrib/operators/mysql_to_gcs.html | 508 +
.../operators/postgres_to_gcs_operator.html | 475 +
.../airflow/contrib/operators/pubsub_operator.html | 652 +
.../contrib/operators/qubole_check_operator.html | 444 +
.../airflow/contrib/operators/qubole_operator.html | 395 +
.../contrib/operators/s3_list_operator.html | 313 +
.../contrib/operators/s3_to_gcs_operator.html | 424 +
.../operators/s3_to_gcs_transfer_operator.html | 356 +
.../contrib/operators/sagemaker_base_operator.html | 319 +
.../sagemaker_endpoint_config_operator.html | 285 +
.../operators/sagemaker_endpoint_operator.html | 369 +
.../operators/sagemaker_model_operator.html | 286 +
.../operators/sagemaker_training_operator.html | 317 +
.../operators/sagemaker_transform_operator.html | 343 +
.../operators/sagemaker_tuning_operator.html | 318 +
.../airflow/contrib/operators/sftp_operator.html | 399 +
.../contrib/operators/slack_webhook_operator.html | 311 +
.../contrib/operators/snowflake_operator.html | 286 +
.../contrib/operators/spark_jdbc_operator.html | 433 +
.../contrib/operators/spark_sql_operator.html | 330 +
.../contrib/operators/spark_submit_operator.html | 390 +
.../airflow/contrib/operators/sqoop_operator.html | 457 +
.../airflow/contrib/operators/ssh_operator.html | 392 +
.../contrib/operators/vertica_operator.html | 268 +
.../airflow/contrib/operators/vertica_to_hive.html | 358 +
.../airflow/contrib/operators/winrm_operator.html | 329 +
.../airflow/contrib/sensors/aws_athena_sensor.html | 298 +
.../sensors/aws_glue_catalog_partition_sensor.html | 312 +
.../sensors/aws_redshift_cluster_sensor.html | 271 +
.../contrib/sensors/azure_cosmos_sensor.html | 286 +
.../airflow/contrib/sensors/bash_sensor.html | 314 +
.../airflow/contrib/sensors/bigquery_sensor.html | 289 +
.../contrib/sensors/cassandra_record_sensor.html | 280 +
.../contrib/sensors/cassandra_table_sensor.html | 275 +
.../airflow/contrib/sensors/datadog_sensor.html | 302 +
.../airflow/contrib/sensors/emr_base_sensor.html | 275 +
.../contrib/sensors/emr_job_flow_sensor.html | 274 +
.../airflow/contrib/sensors/emr_step_sensor.html | 277 +
.../airflow/contrib/sensors/file_sensor.html | 290 +
.../airflow/contrib/sensors/ftp_sensor.html | 319 +
.../airflow/contrib/sensors/gcs_sensor.html | 385 +
.../airflow/contrib/sensors/hdfs_sensor.html | 297 +
.../airflow/contrib/sensors/jira_sensor.html | 370 +
.../airflow/contrib/sensors/pubsub_sensor.html | 324 +
.../airflow/contrib/sensors/python_sensor.html | 300 +
.../airflow/contrib/sensors/qubole_sensor.html | 339 +
.../airflow/contrib/sensors/redis_key_sensor.html | 266 +
.../contrib/sensors/sagemaker_base_sensor.html | 293 +
.../contrib/sensors/sagemaker_endpoint_sensor.html | 280 +
.../contrib/sensors/sagemaker_training_sensor.html | 321 +
.../sensors/sagemaker_transform_sensor.html | 281 +
.../contrib/sensors/sagemaker_tuning_sensor.html | 281 +
.../airflow/contrib/sensors/sftp_sensor.html | 271 +
.../airflow/contrib/sensors/wasb_sensor.html | 318 +
.../airflow/contrib/sensors/weekday_sensor.html | 322 +
.../airflow/executors/celery_executor.html | 338 +
.../_modules/airflow/executors/local_executor.html | 450 +
.../airflow/executors/sequential_executor.html | 276 +
1.10.2/_modules/airflow/hooks/S3_hook.html | 762 +
1.10.2/_modules/airflow/hooks/dbapi_hook.html | 522 +
1.10.2/_modules/airflow/hooks/druid_hook.html | 384 +
1.10.2/_modules/airflow/hooks/hdfs_hook.html | 320 +
1.10.2/_modules/airflow/hooks/hive_hooks.html | 1148 ++
1.10.2/_modules/airflow/hooks/http_hook.html | 422 +
1.10.2/_modules/airflow/hooks/mssql_hook.html | 271 +
1.10.2/_modules/airflow/hooks/mysql_hook.html | 363 +
1.10.2/_modules/airflow/hooks/pig_hook.html | 313 +
1.10.2/_modules/airflow/hooks/postgres_hook.html | 332 +
1.10.2/_modules/airflow/hooks/presto_hook.html | 359 +
1.10.2/_modules/airflow/hooks/samba_hook.html | 271 +
1.10.2/_modules/airflow/hooks/slack_hook.html | 281 +
1.10.2/_modules/airflow/hooks/sqlite_hook.html | 260 +
1.10.2/_modules/airflow/hooks/zendesk_hook.html | 331 +
1.10.2/_modules/airflow/macros.html | 312 +
1.10.2/_modules/airflow/macros/hive.html | 339 +
1.10.2/_modules/airflow/models.html | 5743 +++++++
.../_modules/airflow/operators/bash_operator.html | 357 +
.../_modules/airflow/operators/check_operator.html | 481 +
.../airflow/operators/dagrun_operator.html | 317 +
.../airflow/operators/druid_check_operator.html | 310 +
.../_modules/airflow/operators/dummy_operator.html | 257 +
.../_modules/airflow/operators/email_operator.html | 297 +
.../airflow/operators/generic_transfer.html | 300 +
.../_modules/airflow/operators/hive_operator.html | 361 +
.../airflow/operators/hive_stats_operator.html | 408 +
.../_modules/airflow/operators/hive_to_druid.html | 465 +
.../_modules/airflow/operators/hive_to_mysql.html | 333 +
.../airflow/operators/hive_to_samba_operator.html | 286 +
.../_modules/airflow/operators/http_operator.html | 320 +
.../airflow/operators/latest_only_operator.html | 283 +
.../_modules/airflow/operators/mssql_operator.html | 277 +
.../_modules/airflow/operators/mssql_to_hive.html | 358 +
.../_modules/airflow/operators/mysql_operator.html | 280 +
.../_modules/airflow/operators/mysql_to_hive.html | 361 +
.../_modules/airflow/operators/pig_operator.html | 292 +
.../airflow/operators/postgres_operator.html | 281 +
.../airflow/operators/presto_check_operator.html | 346 +
.../airflow/operators/presto_to_mysql.html | 299 +
.../airflow/operators/python_operator.html | 620 +
.../airflow/operators/redshift_to_s3_operator.html | 370 +
.../operators/s3_file_transform_operator.html | 386 +
.../airflow/operators/s3_to_hive_operator.html | 515 +
.../airflow/operators/s3_to_redshift_operator.html | 327 +
.../_modules/airflow/operators/slack_operator.html | 357 +
.../airflow/operators/sqlite_operator.html | 271 +
.../airflow/operators/subdag_operator.html | 322 +
.../airflow/sensors/base_sensor_operator.html | 350 +
.../airflow/sensors/external_task_sensor.html | 320 +
1.10.2/_modules/airflow/sensors/hdfs_sensor.html | 338 +
.../airflow/sensors/hive_partition_sensor.html | 294 +
1.10.2/_modules/airflow/sensors/http_sensor.html | 311 +
.../sensors/metastore_partition_sensor.html | 302 +
.../sensors/named_hive_partition_sensor.html | 323 +
1.10.2/_modules/airflow/sensors/s3_key_sensor.html | 312 +
.../_modules/airflow/sensors/s3_prefix_sensor.html | 301 +
1.10.2/_modules/airflow/sensors/sql_sensor.html | 277 +
.../airflow/sensors/time_delta_sensor.html | 265 +
1.10.2/_modules/airflow/sensors/time_sensor.html | 259 +
.../_modules/airflow/sensors/web_hdfs_sensor.html | 263 +
1.10.2/_modules/index.html | 431 +
1.10.2/_sources/api.rst.txt | 143 +
1.10.2/_sources/changelog.rst.txt | 21 +
1.10.2/_sources/cli.rst.txt | 28 +
1.10.2/_sources/code.rst.txt | 483 +
1.10.2/_sources/concepts.rst.txt | 901 ++
1.10.2/_sources/faq.rst.txt | 201 +
1.10.2/_sources/howto/check-health.rst.txt | 47 +
1.10.2/_sources/howto/executor/use-celery.rst.txt | 71 +
1.10.2/_sources/howto/executor/use-dask.rst.txt | 50 +
1.10.2/_sources/howto/executor/use-mesos.rst.txt | 82 +
1.10.2/_sources/howto/index.rst.txt | 42 +
1.10.2/_sources/howto/initialize-database.rst.txt | 48 +
1.10.2/_sources/howto/manage-connections.rst.txt | 400 +
1.10.2/_sources/howto/operator.rst.txt | 1657 ++
1.10.2/_sources/howto/run-with-systemd.rst.txt | 32 +
1.10.2/_sources/howto/run-with-upstart.rst.txt | 37 +
1.10.2/_sources/howto/secure-connections.rst.txt | 50 +
1.10.2/_sources/howto/set-config.rst.txt | 66 +
1.10.2/_sources/howto/use-test-config.rst.txt | 34 +
1.10.2/_sources/howto/write-logs.rst.txt | 138 +
1.10.2/_sources/index.rst.txt | 96 +
1.10.2/_sources/installation.rst.txt | 143 +
1.10.2/_sources/integration.rst.txt | 1419 ++
1.10.2/_sources/kubernetes.rst.txt | 132 +
1.10.2/_sources/license.rst.txt | 199 +
1.10.2/_sources/lineage.rst.txt | 102 +
1.10.2/_sources/metrics.rst.txt | 67 +
1.10.2/_sources/plugins.rst.txt | 253 +
1.10.2/_sources/profiling.rst.txt | 58 +
1.10.2/_sources/project.rst.txt | 74 +
1.10.2/_sources/scheduler.rst.txt | 186 +
1.10.2/_sources/security.rst.txt | 439 +
1.10.2/_sources/start.rst.txt | 71 +
1.10.2/_sources/timezone.rst.txt | 163 +
1.10.2/_sources/tutorial.rst.txt | 470 +
1.10.2/_sources/ui.rst.txt | 119 +
1.10.2/_static/ajax-loader.gif | Bin 0 -> 673 bytes
1.10.2/_static/basic.css | 676 +
1.10.2/_static/comment-bright.png | Bin 0 -> 756 bytes
1.10.2/_static/comment-close.png | Bin 0 -> 829 bytes
1.10.2/_static/comment.png | Bin 0 -> 641 bytes
1.10.2/_static/css/badge_only.css | 1 +
1.10.2/_static/css/theme.css | 6 +
1.10.2/_static/doctools.js | 315 +
1.10.2/_static/documentation_options.js | 10 +
1.10.2/_static/down-pressed.png | Bin 0 -> 222 bytes
1.10.2/_static/down.png | Bin 0 -> 202 bytes
1.10.2/_static/file.png | Bin 0 -> 286 bytes
1.10.2/_static/fonts/Inconsolata-Bold.ttf | Bin 0 -> 109948 bytes
1.10.2/_static/fonts/Inconsolata-Regular.ttf | Bin 0 -> 96964 bytes
1.10.2/_static/fonts/Inconsolata.ttf | Bin 0 -> 63184 bytes
1.10.2/_static/fonts/Lato-Bold.ttf | Bin 0 -> 656544 bytes
1.10.2/_static/fonts/Lato-Regular.ttf | Bin 0 -> 656568 bytes
1.10.2/_static/fonts/Lato/lato-bold.eot | Bin 0 -> 256056 bytes
1.10.2/_static/fonts/Lato/lato-bold.ttf | Bin 0 -> 600856 bytes
1.10.2/_static/fonts/Lato/lato-bold.woff | Bin 0 -> 309728 bytes
1.10.2/_static/fonts/Lato/lato-bold.woff2 | Bin 0 -> 184912 bytes
1.10.2/_static/fonts/Lato/lato-bolditalic.eot | Bin 0 -> 266158 bytes
1.10.2/_static/fonts/Lato/lato-bolditalic.ttf | Bin 0 -> 622572 bytes
1.10.2/_static/fonts/Lato/lato-bolditalic.woff | Bin 0 -> 323344 bytes
1.10.2/_static/fonts/Lato/lato-bolditalic.woff2 | Bin 0 -> 193308 bytes
1.10.2/_static/fonts/Lato/lato-italic.eot | Bin 0 -> 268604 bytes
1.10.2/_static/fonts/Lato/lato-italic.ttf | Bin 0 -> 639388 bytes
1.10.2/_static/fonts/Lato/lato-italic.woff | Bin 0 -> 328412 bytes
1.10.2/_static/fonts/Lato/lato-italic.woff2 | Bin 0 -> 195704 bytes
1.10.2/_static/fonts/Lato/lato-regular.eot | Bin 0 -> 253461 bytes
1.10.2/_static/fonts/Lato/lato-regular.ttf | Bin 0 -> 607720 bytes
1.10.2/_static/fonts/Lato/lato-regular.woff | Bin 0 -> 309192 bytes
1.10.2/_static/fonts/Lato/lato-regular.woff2 | Bin 0 -> 182708 bytes
1.10.2/_static/fonts/RobotoSlab-Bold.ttf | Bin 0 -> 170616 bytes
1.10.2/_static/fonts/RobotoSlab-Regular.ttf | Bin 0 -> 169064 bytes
.../fonts/RobotoSlab/roboto-slab-v7-bold.eot | Bin 0 -> 79520 bytes
.../fonts/RobotoSlab/roboto-slab-v7-bold.ttf | Bin 0 -> 170616 bytes
.../fonts/RobotoSlab/roboto-slab-v7-bold.woff | Bin 0 -> 87624 bytes
.../fonts/RobotoSlab/roboto-slab-v7-bold.woff2 | Bin 0 -> 67312 bytes
.../fonts/RobotoSlab/roboto-slab-v7-regular.eot | Bin 0 -> 78331 bytes
.../fonts/RobotoSlab/roboto-slab-v7-regular.ttf | Bin 0 -> 169064 bytes
.../fonts/RobotoSlab/roboto-slab-v7-regular.woff | Bin 0 -> 86288 bytes
.../fonts/RobotoSlab/roboto-slab-v7-regular.woff2 | Bin 0 -> 66444 bytes
1.10.2/_static/fonts/fontawesome-webfont.eot | Bin 0 -> 165742 bytes
1.10.2/_static/fonts/fontawesome-webfont.svg | 2671 ++++
1.10.2/_static/fonts/fontawesome-webfont.ttf | Bin 0 -> 165548 bytes
1.10.2/_static/fonts/fontawesome-webfont.woff | Bin 0 -> 98024 bytes
1.10.2/_static/fonts/fontawesome-webfont.woff2 | Bin 0 -> 77160 bytes
1.10.2/_static/jquery-3.2.1.js | 10253 +++++++++++++
1.10.2/_static/jquery.js | 4 +
1.10.2/_static/js/modernizr.min.js | 4 +
1.10.2/_static/js/theme.js | 3 +
1.10.2/_static/language_data.js | 297 +
1.10.2/_static/minus.png | Bin 0 -> 90 bytes
1.10.2/_static/plus.png | Bin 0 -> 90 bytes
1.10.2/_static/pygments.css | 69 +
1.10.2/_static/searchtools.js | 481 +
1.10.2/_static/underscore-1.3.1.js | 999 ++
1.10.2/_static/underscore.js | 31 +
1.10.2/_static/up-pressed.png | Bin 0 -> 214 bytes
1.10.2/_static/up.png | Bin 0 -> 203 bytes
1.10.2/_static/websupport.js | 808 +
1.10.2/api.html | 361 +
1.10.2/changelog.html | 2587 ++++
1.10.2/cli.html | 1750 +++
1.10.2/code.html | 15156 +++++++++++++++++++
1.10.2/concepts.html | 1035 ++
1.10.2/faq.html | 403 +
1.10.2/genindex.html | 2230 +++
1.10.2/howto/check-health.html | 282 +
1.10.2/howto/executor/use-celery.html | 293 +
1.10.2/howto/executor/use-dask.html | 275 +
1.10.2/howto/executor/use-mesos.html | 314 +
1.10.2/howto/index.html | 297 +
1.10.2/howto/initialize-database.html | 280 +
1.10.2/howto/manage-connections.html | 597 +
1.10.2/howto/operator.html | 2514 +++
1.10.2/howto/run-with-systemd.html | 262 +
1.10.2/howto/run-with-upstart.html | 264 +
1.10.2/howto/secure-connections.html | 278 +
1.10.2/howto/set-config.html | 290 +
1.10.2/howto/use-test-config.html | 263 +
1.10.2/howto/write-logs.html | 376 +
1.10.2/http-routingtable.html | 295 +
1.10.2/index.html | 937 ++
1.10.2/installation.html | 425 +
1.10.2/integration.html | 9075 +++++++++++
1.10.2/kubernetes.html | 403 +
1.10.2/license.html | 411 +
1.10.2/lineage.html | 315 +
1.10.2/metrics.html | 335 +
1.10.2/objects.inv | Bin 0 -> 9678 bytes
1.10.2/plugins.html | 457 +
1.10.2/profiling.html | 272 +
1.10.2/project.html | 288 +
1.10.2/py-modindex.html | 252 +
1.10.2/scheduler.html | 413 +
1.10.2/search.html | 238 +
1.10.2/searchindex.js | 1 +
1.10.2/security.html | 628 +
1.10.2/start.html | 283 +
1.10.2/timezone.html | 365 +
1.10.2/tutorial.html | 667 +
1.10.2/ui.html | 318 +
379 files changed, 162695 insertions(+)
diff --git a/1.10.2/.buildinfo b/1.10.2/.buildinfo
new file mode 100644
index 0000000..2cd3f36
--- /dev/null
+++ b/1.10.2/.buildinfo
@@ -0,0 +1,4 @@
+# Sphinx build info version 1
+# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
+config: 31d38953a908168368629ac053123a0f
+tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/1.10.2/_images/adhoc.png b/1.10.2/_images/adhoc.png
new file mode 100644
index 0000000..77ea780
Binary files /dev/null and b/1.10.2/_images/adhoc.png differ
diff --git a/1.10.2/_images/airflow.gif b/1.10.2/_images/airflow.gif
new file mode 100644
index 0000000..7646e37
Binary files /dev/null and b/1.10.2/_images/airflow.gif differ
diff --git a/1.10.2/_images/apache.jpg b/1.10.2/_images/apache.jpg
new file mode 100644
index 0000000..312251f
Binary files /dev/null and b/1.10.2/_images/apache.jpg differ
diff --git a/1.10.2/_images/branch_bad.png b/1.10.2/_images/branch_bad.png
new file mode 100644
index 0000000..586844f
Binary files /dev/null and b/1.10.2/_images/branch_bad.png differ
diff --git a/1.10.2/_images/branch_good.png b/1.10.2/_images/branch_good.png
new file mode 100644
index 0000000..fbd4650
Binary files /dev/null and b/1.10.2/_images/branch_good.png differ
diff --git a/1.10.2/_images/chart.png b/1.10.2/_images/chart.png
new file mode 100644
index 0000000..bfca26b
Binary files /dev/null and b/1.10.2/_images/chart.png differ
diff --git a/1.10.2/_images/chart_form.png b/1.10.2/_images/chart_form.png
new file mode 100644
index 0000000..f73daf5
Binary files /dev/null and b/1.10.2/_images/chart_form.png differ
diff --git a/1.10.2/_images/code.png b/1.10.2/_images/code.png
new file mode 100644
index 0000000..60e8f58
Binary files /dev/null and b/1.10.2/_images/code.png differ
diff --git a/1.10.2/_images/connection_create.png b/1.10.2/_images/connection_create.png
new file mode 100644
index 0000000..8a574d4
Binary files /dev/null and b/1.10.2/_images/connection_create.png differ
diff --git a/1.10.2/_images/connection_edit.png b/1.10.2/_images/connection_edit.png
new file mode 100644
index 0000000..c6d14da
Binary files /dev/null and b/1.10.2/_images/connection_edit.png differ
diff --git a/1.10.2/_images/connections.png b/1.10.2/_images/connections.png
new file mode 100644
index 0000000..3a28473
Binary files /dev/null and b/1.10.2/_images/connections.png differ
diff --git a/1.10.2/_images/context.png b/1.10.2/_images/context.png
new file mode 100644
index 0000000..bac718f
Binary files /dev/null and b/1.10.2/_images/context.png differ
diff --git a/1.10.2/_images/dags.png b/1.10.2/_images/dags.png
new file mode 100644
index 0000000..04c8213
Binary files /dev/null and b/1.10.2/_images/dags.png differ
diff --git a/1.10.2/_images/duration.png b/1.10.2/_images/duration.png
new file mode 100644
index 0000000..b59fc09
Binary files /dev/null and b/1.10.2/_images/duration.png differ
diff --git a/1.10.2/_images/gantt.png b/1.10.2/_images/gantt.png
new file mode 100644
index 0000000..d74e1e5
Binary files /dev/null and b/1.10.2/_images/gantt.png differ
diff --git a/1.10.2/_images/graph.png b/1.10.2/_images/graph.png
new file mode 100644
index 0000000..ea2b705
Binary files /dev/null and b/1.10.2/_images/graph.png differ
diff --git a/1.10.2/_images/latest_only_with_trigger.png b/1.10.2/_images/latest_only_with_trigger.png
new file mode 100644
index 0000000..629adfa
Binary files /dev/null and b/1.10.2/_images/latest_only_with_trigger.png differ
diff --git a/1.10.2/_images/pin_large.png b/1.10.2/_images/pin_large.png
new file mode 100644
index 0000000..986c88b
Binary files /dev/null and b/1.10.2/_images/pin_large.png differ
diff --git a/1.10.2/_images/subdag_after.png b/1.10.2/_images/subdag_after.png
new file mode 100644
index 0000000..166a6de
Binary files /dev/null and b/1.10.2/_images/subdag_after.png differ
diff --git a/1.10.2/_images/subdag_before.png b/1.10.2/_images/subdag_before.png
new file mode 100644
index 0000000..ebc3e58
Binary files /dev/null and b/1.10.2/_images/subdag_before.png differ
diff --git a/1.10.2/_images/subdag_zoom.png b/1.10.2/_images/subdag_zoom.png
new file mode 100644
index 0000000..08fcf5c
Binary files /dev/null and b/1.10.2/_images/subdag_zoom.png differ
diff --git a/1.10.2/_images/tree.png b/1.10.2/_images/tree.png
new file mode 100644
index 0000000..06750aa
Binary files /dev/null and b/1.10.2/_images/tree.png differ
diff --git a/1.10.2/_images/variable_hidden.png b/1.10.2/_images/variable_hidden.png
new file mode 100644
index 0000000..e081ca3
Binary files /dev/null and b/1.10.2/_images/variable_hidden.png differ
diff --git a/1.10.2/_modules/airflow/contrib/executors/kubernetes_executor.html b/1.10.2/_modules/airflow/contrib/executors/kubernetes_executor.html
new file mode 100644
index 0000000..69d7e74
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/executors/kubernetes_executor.html
@@ -0,0 +1,897 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.executors.kubernetes_executor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+ <div class="version">
+ 1.10.2
+ </div>
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.executors.kubernetes_executor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.executors.kubernetes_executor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">base64</span>
+<span class="kn">import</span> <span class="nn">json</span>
+<span class="kn">import</span> <span class="nn">multiprocessing</span>
+<span class="kn">from</span> <span class="nn">queue</span> <span class="k">import</span> <span class="n">Queue</span>
+<span class="kn">from</span> <span class="nn">dateutil</span> <span class="k">import</span> <span class="n">parser</span>
+<span class="kn">from</span> <span class="nn">uuid</span> <span class="k">import</span> <span class="n">uuid4</span>
+<span class="kn">import</span> <span class="nn">kubernetes</span>
+<span class="kn">from</span> <span class="nn">kubernetes</span> <span class="k">import</span> <span class="n">watch</span><span class="p">,</span> <span class="n">client</span>
+<span class="kn">from</span> <span class="nn">kubernetes.client.rest</span> <span class="k">import</span> <span class="n">ApiException</span>
+<span class="kn">from</span> <span class="nn">airflow.configuration</span> <span class="k">import</span> <span class="n">conf</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.kubernetes.pod_launcher</span> <span class="k">import</span> <span class="n">PodLauncher</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.kubernetes.kube_client</span> <span class="k">import</span> <span class="n">get_kube_client</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.kubernetes.worker_configuration</span> <span class="k">import</span> <span class="n">WorkerConfiguration</span>
+<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="k">import</span> <span class="n">BaseExecutor</span>
+<span class="kn">from</span> <span class="nn">airflow.executors</span> <span class="k">import</span> <span class="n">Executors</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">TaskInstance</span><span class="p">,</span> <span class="n">KubeResourceVersion</span><span class="p">,</span> <span class="n">KubeWorkerIdentifier</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.db</span> <span class="k">import</span> <span class="n">provide_session</span><span class="p">,</span> <span class="n">create_session</span>
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">configuration</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowConfigException</span><span class="p">,</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+
+<span class="k">class</span> <span class="nc">KubernetesExecutorConfig</span><span class="p">:</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">image</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">image_pull_policy</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">request_memory</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">request_cpu</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">limit_memory</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">limit_cpu</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">gcp_service_account_key</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">node_selectors</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">affinity</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">annotations</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">volumes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">volume_mounts</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">tolerations</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">image</span> <span class="o">=</span> <span class="n">image</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">image_pull_policy</span> <span class="o">=</span> <span class="n">image_pull_policy</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">request_memory</span> <span class="o">=</span> <span class="n">request_memory</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">request_cpu</span> <span class="o">=</span> <span class="n">request_cpu</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">limit_memory</span> <span class="o">=</span> <span class="n">limit_memory</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">limit_cpu</span> <span class="o">=</span> <span class="n">limit_cpu</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">gcp_service_account_key</span> <span class="o">=</span> <span class="n">gcp_service_account_key</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">node_selectors</span> <span class="o">=</span> <span class="n">node_selectors</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">affinity</span> <span class="o">=</span> <span class="n">affinity</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">annotations</span> <span class="o">=</span> <span class="n">annotations</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">volumes</span> <span class="o">=</span> <span class="n">volumes</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">volume_mounts</span> <span class="o">=</span> <span class="n">volume_mounts</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">tolerations</span> <span class="o">=</span> <span class="n">tolerations</span>
+
+ <span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="s2">"</span><span class="si">{}</span><span class="s2">(image=</span><span class="si">{}</span><span class="s2">, image_pull_policy=</span><span class="si">{}</span><span class="s2">, request_memory=</span><span class="si">{}</span><span class="s2">, request_cpu=</span><span class="si">{}</span><span class="s2">, "</span> \
+ <span class="s2">"limit_memory=</span><span class="si">{}</span><span class="s2">, limit_cpu=</span><span class="si">{}</span><span class="s2">, gcp_service_account_key=</span><span class="si">{}</span><span class="s2">, "</span> \
+ <span class="s2">"node_selectors=</span><span class="si">{}</span><span class="s2">, affinity=</span><span class="si">{}</span><span class="s2">, annotations=</span><span class="si">{}</span><span class="s2">, volumes=</span><span class="si">{}</span><span class="s2">, "</span> \
+ <span class="s2">"volume_mounts=</span><span class="si">{}</span><span class="s2">, tolerations=</span><span class="si">{}</span><span class="s2">)"</span> \
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">KubernetesExecutorConfig</span><span class="o">.</span><span class="vm">__name__</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">image</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">image_pull_policy</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">request_memory</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">request_cpu</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">limit_memory</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">limit_cpu</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">gcp_service_account_key</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">node_selectors</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">affinity</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">annotations</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">volumes</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">volume_mounts</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">tolerations</span><span class="p">)</span>
+
+ <span class="nd">@staticmethod</span>
+ <span class="k">def</span> <span class="nf">from_dict</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">obj</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">KubernetesExecutorConfig</span><span class="p">()</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span>
+ <span class="s1">'Cannot convert a non-dictionary object into a KubernetesExecutorConfig'</span><span class="p">)</span>
+
+ <span class="n">namespaced</span> <span class="o">=</span> <span class="n">obj</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">Executors</span><span class="o">.</span><span class="n">KubernetesExecutor</span><span class="p">,</span> <span class="p">{})</span>
+
+ <span class="k">return</span> <span class="n">KubernetesExecutorConfig</span><span class="p">(</span>
+ <span class="n">image</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'image'</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
+ <span class="n">image_pull_policy</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'image_pull_policy'</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
+ <span class="n">request_memory</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'request_memory'</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
+ <span class="n">request_cpu</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'request_cpu'</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
+ <span class="n">limit_memory</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'limit_memory'</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
+ <span class="n">limit_cpu</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'limit_cpu'</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
+ <span class="n">gcp_service_account_key</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'gcp_service_account_key'</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
+ <span class="n">node_selectors</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'node_selectors'</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
+ <span class="n">affinity</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'affinity'</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
+ <span class="n">annotations</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'annotations'</span><span class="p">,</span> <span class="p">{}),</span>
+ <span class="n">volumes</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'volumes'</span><span class="p">,</span> <span class="p">[]),</span>
+ <span class="n">volume_mounts</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'volume_mounts'</span><span class="p">,</span> <span class="p">[]),</span>
+ <span class="n">tolerations</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'tolerations'</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
+ <span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">as_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="p">{</span>
+ <span class="s1">'image'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">image</span><span class="p">,</span>
+ <span class="s1">'image_pull_policy'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">image_pull_policy</span><span class="p">,</span>
+ <span class="s1">'request_memory'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">request_memory</span><span class="p">,</span>
+ <span class="s1">'request_cpu'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">request_cpu</span><span class="p">,</span>
+ <span class="s1">'limit_memory'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">limit_memory</span><span class="p">,</span>
+ <span class="s1">'limit_cpu'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">limit_cpu</span><span class="p">,</span>
+ <span class="s1">'gcp_service_account_key'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">gcp_service_account_key</span><span class="p">,</span>
+ <span class="s1">'node_selectors'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">node_selectors</span><span class="p">,</span>
+ <span class="s1">'affinity'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">affinity</span><span class="p">,</span>
+ <span class="s1">'annotations'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">annotations</span><span class="p">,</span>
+ <span class="s1">'volumes'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">volumes</span><span class="p">,</span>
+ <span class="s1">'volume_mounts'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">volume_mounts</span><span class="p">,</span>
+ <span class="s1">'tolerations'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">tolerations</span><span class="p">,</span>
+ <span class="p">}</span>
+
+
+<span class="k">class</span> <span class="nc">KubeConfig</span><span class="p">:</span>
+ <span class="n">core_section</span> <span class="o">=</span> <span class="s1">'core'</span>
+ <span class="n">kubernetes_section</span> <span class="o">=</span> <span class="s1">'kubernetes'</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="n">configuration_dict</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">as_dict</span><span class="p">(</span><span class="n">display_sensitive</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">core_configuration</span> <span class="o">=</span> <span class="n">configuration_dict</span><span class="p">[</span><span class="s1">'core'</span><span class="p">]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_secrets</span> <span class="o">=</span> <span class="n">configuration_dict</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'kubernetes_secrets'</span><span class="p">,</span> <span class="p">{})</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">airflow_home</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">core_section</span><span class="p">,</span> <span class="s1">'airflow_home'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">dags_folder</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">core_section</span><span class="p">,</span> <span class="s1">'dags_folder'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">parallelism</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">core_section</span><span class="p">,</span> <span class="s1">'PARALLELISM'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">worker_container_repository</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'worker_container_repository'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">worker_container_tag</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'worker_container_tag'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_image</span> <span class="o">=</span> <span class="s1">'</span><span class="si">{}</span><span class="s1">:</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">worker_container_repository</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_container_tag</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_image_pull_policy</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s2">"worker_container_image_pull_policy"</span>
+ <span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_node_selectors</span> <span class="o">=</span> <span class="n">configuration_dict</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'kubernetes_node_selectors'</span><span class="p">,</span> <span class="p">{})</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delete_worker_pods</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'delete_worker_pods'</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">worker_service_account_name</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'worker_service_account_name'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">image_pull_secrets</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'image_pull_secrets'</span><span class="p">)</span>
+
+ <span class="c1"># NOTE: user can build the dags into the docker image directly,</span>
+ <span class="c1"># this will set to True if so</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">dags_in_image</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'dags_in_image'</span><span class="p">)</span>
+
+ <span class="c1"># NOTE: `git_repo` and `git_branch` must be specified together as a pair</span>
+ <span class="c1"># The http URL of the git repository to clone from</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">git_repo</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'git_repo'</span><span class="p">)</span>
+ <span class="c1"># The branch of the repository to be checked out</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">git_branch</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'git_branch'</span><span class="p">)</span>
+ <span class="c1"># Optionally, the directory in the git repository containing the dags</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">git_subpath</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'git_subpath'</span><span class="p">)</span>
+ <span class="c1"># Optionally, the root directory for git operations</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">git_sync_root</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'git_sync_root'</span><span class="p">)</span>
+ <span class="c1"># Optionally, the name at which to publish the checked-out files under --root</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">git_sync_dest</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'git_sync_dest'</span><span class="p">)</span>
+ <span class="c1"># Optionally, if git_dags_folder_mount_point is set the worker will use</span>
+ <span class="c1"># {git_dags_folder_mount_point}/{git_sync_dest}/{git_subpath} as dags_folder</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">git_dags_folder_mount_point</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span>
+ <span class="s1">'git_dags_folder_mount_point'</span><span class="p">)</span>
+
+ <span class="c1"># Optionally a user may supply a `git_user` and `git_password` for private</span>
+ <span class="c1"># repositories</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">git_user</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'git_user'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">git_password</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'git_password'</span><span class="p">)</span>
+
+ <span class="c1"># NOTE: The user may optionally use a volume claim to mount a PV containing</span>
+ <span class="c1"># DAGs directly</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">dags_volume_claim</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'dags_volume_claim'</span><span class="p">)</span>
+
+ <span class="c1"># This prop may optionally be set for PV Claims and is used to write logs</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">logs_volume_claim</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'logs_volume_claim'</span><span class="p">)</span>
+
+ <span class="c1"># This prop may optionally be set for PV Claims and is used to locate DAGs</span>
+ <span class="c1"># on a SubPath</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">dags_volume_subpath</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'dags_volume_subpath'</span><span class="p">)</span>
+
+ <span class="c1"># This prop may optionally be set for PV Claims and is used to locate logs</span>
+ <span class="c1"># on a SubPath</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">logs_volume_subpath</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'logs_volume_subpath'</span><span class="p">)</span>
+
+ <span class="c1"># Optionally, hostPath volume containing DAGs</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">dags_volume_host</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'dags_volume_host'</span><span class="p">)</span>
+
+ <span class="c1"># Optionally, write logs to a hostPath Volume</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">logs_volume_host</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'logs_volume_host'</span><span class="p">)</span>
+
+ <span class="c1"># This prop may optionally be set for PV Claims and is used to write logs</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">base_log_folder</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">core_section</span><span class="p">,</span> <span class="s1">'base_log_folder'</span><span class="p">)</span>
+
+ <span class="c1"># The Kubernetes Namespace in which the Scheduler and Webserver reside. Note</span>
+ <span class="c1"># that if your</span>
+ <span class="c1"># cluster has RBAC enabled, your scheduler may need service account permissions to</span>
+ <span class="c1"># create, watch, get, and delete pods in this namespace.</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_namespace</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'namespace'</span><span class="p">)</span>
+ <span class="c1"># The Kubernetes Namespace in which pods will be created by the executor. Note</span>
+ <span class="c1"># that if your</span>
+ <span class="c1"># cluster has RBAC enabled, your workers may need service account permissions to</span>
+ <span class="c1"># interact with cluster components.</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">executor_namespace</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'namespace'</span><span class="p">)</span>
+ <span class="c1"># Task secrets managed by KubernetesExecutor.</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">gcp_service_account_keys</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span>
+ <span class="s1">'gcp_service_account_keys'</span><span class="p">)</span>
+
+ <span class="c1"># If the user is using the git-sync container to clone their repository via git,</span>
+ <span class="c1"># allow them to specify repository, tag, and pod name for the init container.</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">git_sync_container_repository</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'git_sync_container_repository'</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">git_sync_container_tag</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'git_sync_container_tag'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">git_sync_container</span> <span class="o">=</span> <span class="s1">'</span><span class="si">{}</span><span class="s1">:</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">git_sync_container_repository</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">git_sync_container_tag</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">git_sync_init_container_name</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'git_sync_init_container_name'</span><span class="p">)</span>
+
+ <span class="c1"># The worker pod may optionally have a valid Airflow config loaded via a</span>
+ <span class="c1"># configmap</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">airflow_configmap</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'airflow_configmap'</span><span class="p">)</span>
+
+ <span class="n">affinity_json</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'affinity'</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">affinity_json</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_affinity</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">affinity_json</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_affinity</span> <span class="o">=</span> <span class="kc">None</span>
+
+ <span class="n">tolerations_json</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">'tolerations'</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">tolerations_json</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_tolerations</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">tolerations_json</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_tolerations</span> <span class="o">=</span> <span class="kc">None</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">_validate</span><span class="p">()</span>
+
+ <span class="k">def</span> <span class="nf">_validate</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="c1"># TODO: use XOR for dags_volume_claim and git_dags_folder_mount_point</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags_volume_claim</span> \
+ <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags_volume_host</span> \
+ <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags_in_image</span> \
+ <span class="ow">and</span> <span class="p">(</span><span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">git_repo</span> <span class="ow">or</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">git_branch</span> <span class="ow">or</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">git_dags_folder_mount_point</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="n">AirflowConfigException</span><span class="p">(</span>
+ <span class="s1">'In kubernetes mode the following must be set in the `kubernetes` '</span>
+ <span class="s1">'config section: `dags_volume_claim` '</span>
+ <span class="s1">'or `dags_volume_host` '</span>
+ <span class="s1">'or `dags_in_image` '</span>
+ <span class="s1">'or `git_repo and git_branch and git_dags_folder_mount_point`'</span><span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">KubernetesJobWatcher</span><span class="p">(</span><span class="n">multiprocessing</span><span class="o">.</span><span class="n">Process</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">,</span> <span class="nb">object</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">namespace</span><span class="p">,</span> <span class="n">watcher_queue</span><span class="p">,</span> <span class="n">resource_version</span><span class="p">,</span> <span class="n">worker_uuid</span><span class="p">):</span>
+ <span class="n">multiprocessing</span><span class="o">.</span><span class="n">Process</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">namespace</span> <span class="o">=</span> <span class="n">namespace</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">worker_uuid</span> <span class="o">=</span> <span class="n">worker_uuid</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">watcher_queue</span> <span class="o">=</span> <span class="n">watcher_queue</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">resource_version</span> <span class="o">=</span> <span class="n">resource_version</span>
+
+ <span class="k">def</span> <span class="nf">run</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="n">kube_client</span> <span class="o">=</span> <span class="n">get_kube_client</span><span class="p">()</span>
+ <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">resource_version</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_run</span><span class="p">(</span><span class="n">kube_client</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">resource_version</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">worker_uuid</span><span class="p">)</span>
+ <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s1">'Unknown error in KubernetesJobWatcher. Failing'</span><span class="p">)</span>
+ <span class="k">raise</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s1">'Watch died gracefully, starting back up with: '</span>
+ <span class="s1">'last resource_version: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">resource_version</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_run</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">kube_client</span><span class="p">,</span> <span class="n">resource_version</span><span class="p">,</span> <span class="n">worker_uuid</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'Event: and now my watch begins starting at resource_version: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">resource_version</span>
+ <span class="p">)</span>
+ <span class="n">watcher</span> <span class="o">=</span> <span class="n">watch</span><span class="o">.</span><span class="n">Watch</span><span class="p">()</span>
+
+ <span class="n">kwargs</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'label_selector'</span><span class="p">:</span> <span class="s1">'airflow-worker=</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">worker_uuid</span><span class="p">)}</span>
+ <span class="k">if</span> <span class="n">resource_version</span><span class="p">:</span>
+ <span class="n">kwargs</span><span class="p">[</span><span class="s1">'resource_version'</span><span class="p">]</span> <span class="o">=</span> <span class="n">resource_version</span>
+
+ <span class="n">last_resource_version</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="k">for</span> <span class="n">event</span> <span class="ow">in</span> <span class="n">watcher</span><span class="o">.</span><span class="n">stream</span><span class="p">(</span><span class="n">kube_client</span><span class="o">.</span><span class="n">list_namespaced_pod</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">namespace</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="n">task</span> <span class="o">=</span> <span class="n">event</span><span class="p">[</span><span class="s1">'object'</span><span class="p">]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'Event: </span><span class="si">%s</span><span class="s1"> had an event of type </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">task</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">event</span><span class="p">[</span><span class="s1">'type'</span><span class="p">]</span>
+ <span class="p">)</span>
+ <span class="k">if</span> <span class="n">event</span><span class="p">[</span><span class="s1">'type'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'ERROR'</span><span class="p">:</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">process_error</span><span class="p">(</span><span class="n">event</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">process_status</span><span class="p">(</span>
+ <span class="n">task</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">task</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">phase</span><span class="p">,</span> <span class="n">task</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">labels</span><span class="p">,</span>
+ <span class="n">task</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">resource_version</span>
+ <span class="p">)</span>
+ <span class="n">last_resource_version</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">resource_version</span>
+
+ <span class="k">return</span> <span class="n">last_resource_version</span>
+
+ <span class="k">def</span> <span class="nf">process_error</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">event</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
+ <span class="s1">'Encountered Error response from k8s list namespaced pod stream => </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">event</span>
+ <span class="p">)</span>
+ <span class="n">raw_object</span> <span class="o">=</span> <span class="n">event</span><span class="p">[</span><span class="s1">'raw_object'</span><span class="p">]</span>
+ <span class="k">if</span> <span class="n">raw_object</span><span class="p">[</span><span class="s1">'code'</span><span class="p">]</span> <span class="o">==</span> <span class="mi">410</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'Kubernetes resource version is too old, must reset to 0 => </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">raw_object</span><span class="p">[</span><span class="s1">'message'</span><span class="p">]</span>
+ <span class="p">)</span>
+ <span class="c1"># Return resource version 0</span>
+ <span class="k">return</span> <span class="s1">'0'</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'Kubernetes failure for </span><span class="si">%s</span><span class="s1"> with code </span><span class="si">%s</span><span class="s1"> and message: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">raw_object</span><span class="p">[</span><span class="s1">'reason'</span><span class="p">],</span> <span class="n">raw_object</span><span class="p">[</span><span class="s1">'code'</span><span class="p">],</span> <span class="n">raw_object</span><span class="p">[</span><span class="s1">'message'</span><span class="p">]</span>
+ <span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">process_status</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">pod_id</span><span class="p">,</span> <span class="n">status</span><span class="p">,</span> <span class="n">labels</span><span class="p">,</span> <span class="n">resource_version</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">status</span> <span class="o">==</span> <span class="s1">'Pending'</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Event: </span><span class="si">%s</span><span class="s1"> Pending'</span><span class="p">,</span> <span class="n">pod_id</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="n">status</span> <span class="o">==</span> <span class="s1">'Failed'</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Event: </span><span class="si">%s</span><span class="s1"> Failed'</span><span class="p">,</span> <span class="n">pod_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">watcher_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">pod_id</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">FAILED</span><span class="p">,</span> <span class="n">labels</span><span class="p">,</span> <span class="n">resource_version</span><span class="p">))</span>
+ <span class="k">elif</span> <span class="n">status</span> <span class="o">==</span> <span class="s1">'Succeeded'</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Event: </span><span class="si">%s</span><span class="s1"> Succeeded'</span><span class="p">,</span> <span class="n">pod_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">watcher_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">pod_id</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="n">labels</span><span class="p">,</span> <span class="n">resource_version</span><span class="p">))</span>
+ <span class="k">elif</span> <span class="n">status</span> <span class="o">==</span> <span class="s1">'Running'</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Event: </span><span class="si">%s</span><span class="s1"> is Running'</span><span class="p">,</span> <span class="n">pod_id</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+ <span class="s1">'Event: Invalid state: </span><span class="si">%s</span><span class="s1"> on pod: </span><span class="si">%s</span><span class="s1"> with labels: </span><span class="si">%s</span><span class="s1"> with '</span>
+ <span class="s1">'resource_version: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">status</span><span class="p">,</span> <span class="n">pod_id</span><span class="p">,</span> <span class="n">labels</span><span class="p">,</span> <span class="n">resource_version</span>
+ <span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">AirflowKubernetesScheduler</span><span class="p">(</span><span class="n">LoggingMixin</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">kube_config</span><span class="p">,</span> <span class="n">task_queue</span><span class="p">,</span> <span class="n">result_queue</span><span class="p">,</span> <span class="n">kube_client</span><span class="p">,</span> <span class="n">worker_uuid</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Creating Kubernetes executor"</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span> <span class="o">=</span> <span class="n">kube_config</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span> <span class="o">=</span> <span class="n">task_queue</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="n">result_queue</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">namespace</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span><span class="o">.</span><span class="n">kube_namespace</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Kubernetes using namespace </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">namespace</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_client</span> <span class="o">=</span> <span class="n">kube_client</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">launcher</span> <span class="o">=</span> <span class="n">PodLauncher</span><span class="p">(</span><span class="n">kube_client</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">kube_client</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">worker_configuration</span> <span class="o">=</span> <span class="n">WorkerConfiguration</span><span class="p">(</span><span class="n">kube_config</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">watcher_queue</span> <span class="o">=</span> <span class="n">multiprocessing</span><span class="o">.</span><span class="n">Queue</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">worker_uuid</span> <span class="o">=</span> <span class="n">worker_uuid</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_watcher</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_make_kube_watcher</span><span class="p">()</span>
+
+ <span class="k">def</span> <span class="nf">_make_kube_watcher</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="n">resource_version</span> <span class="o">=</span> <span class="n">KubeResourceVersion</span><span class="o">.</span><span class="n">get_current_resource_version</span><span class="p">()</span>
+ <span class="n">watcher</span> <span class="o">=</span> <span class="n">KubernetesJobWatcher</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">namespace</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">watcher_queue</span><span class="p">,</span>
+ <span class="n">resource_version</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_uuid</span><span class="p">)</span>
+ <span class="n">watcher</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">watcher</span>
+
+ <span class="k">def</span> <span class="nf">_health_check_kube_watcher</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">kube_watcher</span><span class="o">.</span><span class="n">is_alive</span><span class="p">():</span>
+ <span class="k">pass</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
+ <span class="s1">'Error while health checking kube watcher process. '</span>
+ <span class="s1">'Process died for unknown reasons'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_watcher</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_make_kube_watcher</span><span class="p">()</span>
+
+ <span class="k">def</span> <span class="nf">run_next</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">next_job</span><span class="p">):</span>
+ <span class="sd">"""</span>
+
+<span class="sd"> The run_next command will check the task_queue for any un-run jobs.</span>
+<span class="sd"> It will then create a unique job-id, launch that job in the cluster,</span>
+<span class="sd"> and store relevant info in the current_jobs map so we can track the job's</span>
+<span class="sd"> status</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Kubernetes job is </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">next_job</span><span class="p">))</span>
+ <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">kube_executor_config</span> <span class="o">=</span> <span class="n">next_job</span>
+ <span class="n">dag_id</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">execution_date</span><span class="p">,</span> <span class="n">try_number</span> <span class="o">=</span> <span class="n">key</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Kubernetes running for command </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">command</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Kubernetes launching image </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span><span class="o">.</span><span class="n">kube_image</span><span class="p">)</span>
+ <span class="n">pod</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_configuration</span><span class="o">.</span><span class="n">make_pod</span><span class="p">(</span>
+ <span class="n">namespace</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">namespace</span><span class="p">,</span> <span class="n">worker_uuid</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">worker_uuid</span><span class="p">,</span>
+ <span class="n">pod_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_create_pod_id</span><span class="p">(</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">task_id</span><span class="p">),</span>
+ <span class="n">dag_id</span><span class="o">=</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">task_id</span><span class="o">=</span><span class="n">task_id</span><span class="p">,</span> <span class="n">try_number</span><span class="o">=</span><span class="n">try_number</span><span class="p">,</span>
+ <span class="n">execution_date</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_datetime_to_label_safe_datestring</span><span class="p">(</span><span class="n">execution_date</span><span class="p">),</span>
+ <span class="n">airflow_command</span><span class="o">=</span><span class="n">command</span><span class="p">,</span> <span class="n">kube_executor_config</span><span class="o">=</span><span class="n">kube_executor_config</span>
+ <span class="p">)</span>
+ <span class="c1"># the watcher will monitor pods, so we do not block.</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">launcher</span><span class="o">.</span><span class="n">run_pod_async</span><span class="p">(</span><span class="n">pod</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Kubernetes Job created!"</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">delete_pod</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">pod_id</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span><span class="o">.</span><span class="n">delete_worker_pods</span><span class="p">:</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_client</span><span class="o">.</span><span class="n">delete_namespaced_pod</span><span class="p">(</span>
+ <span class="n">pod_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">namespace</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">client</span><span class="o">.</span><span class="n">V1DeleteOptions</span><span class="p">())</span>
+ <span class="k">except</span> <span class="n">ApiException</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="c1"># If the pod is already deleted</span>
+ <span class="k">if</span> <span class="n">e</span><span class="o">.</span><span class="n">status</span> <span class="o">!=</span> <span class="mi">404</span><span class="p">:</span>
+ <span class="k">raise</span>
+
+ <span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> The sync function checks the status of all currently running kubernetes jobs.</span>
+<span class="sd"> If a job is completed, it's status is placed in the result queue to</span>
+<span class="sd"> be sent back to the scheduler.</span>
+
+<span class="sd"> :return:</span>
+
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_health_check_kube_watcher</span><span class="p">()</span>
+ <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">watcher_queue</span><span class="o">.</span><span class="n">empty</span><span class="p">():</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">process_watcher_task</span><span class="p">()</span>
+
+ <span class="k">def</span> <span class="nf">process_watcher_task</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="n">pod_id</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">labels</span><span class="p">,</span> <span class="n">resource_version</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">watcher_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'Attempting to finish pod; pod_id: </span><span class="si">%s</span><span class="s1">; state: </span><span class="si">%s</span><span class="s1">; labels: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">pod_id</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">labels</span>
+ <span class="p">)</span>
+ <span class="n">key</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_labels_to_key</span><span class="p">(</span><span class="n">labels</span><span class="o">=</span><span class="n">labels</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">key</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'finishing job </span><span class="si">%s</span><span class="s1"> - </span><span class="si">%s</span><span class="s1"> (</span><span class="si">%s</span><span class="s1">)'</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">state</span><span class="p">,< [...]
+ <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">pod_id</span><span class="p">,</span> <span class="n">resource_version</span><span class="p">))</span>
+
+ <span class="nd">@staticmethod</span>
+ <span class="k">def</span> <span class="nf">_strip_unsafe_kubernetes_special_chars</span><span class="p">(</span><span class="n">string</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Kubernetes only supports lowercase alphanumeric characters and "-" and "." in</span>
+<span class="sd"> the pod name</span>
+<span class="sd"> However, there are special rules about how "-" and "." can be used so let's</span>
+<span class="sd"> only keep</span>
+<span class="sd"> alphanumeric chars see here for detail:</span>
+<span class="sd"> https://kubernetes.io/docs/concepts/overview/working-with-objects/names/</span>
+
+<span class="sd"> :param string: The requested Pod name</span>
+<span class="sd"> :return: ``str`` Pod name stripped of any unsafe characters</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="s1">''</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">ch</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="k">for</span> <span class="n">ind</span><span class="p">,</span> <span class="n">ch</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">string</span><span class="p">)</span> [...]
+
+ <span class="nd">@staticmethod</span>
+ <span class="k">def</span> <span class="nf">_make_safe_pod_id</span><span class="p">(</span><span class="n">safe_dag_id</span><span class="p">,</span> <span class="n">safe_task_id</span><span class="p">,</span> <span class="n">safe_uuid</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Kubernetes pod names must be <= 253 chars and must pass the following regex for</span>
+<span class="sd"> validation</span>
+<span class="sd"> "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$"</span>
+
+<span class="sd"> :param safe_dag_id: a dag_id with only alphanumeric characters</span>
+<span class="sd"> :param safe_task_id: a task_id with only alphanumeric characters</span>
+<span class="sd"> :param random_uuid: a uuid</span>
+<span class="sd"> :return: ``str`` valid Pod name of appropriate length</span>
+<span class="sd"> """</span>
+ <span class="n">MAX_POD_ID_LEN</span> <span class="o">=</span> <span class="mi">253</span>
+
+ <span class="n">safe_key</span> <span class="o">=</span> <span class="n">safe_dag_id</span> <span class="o">+</span> <span class="n">safe_task_id</span>
+
+ <span class="n">safe_pod_id</span> <span class="o">=</span> <span class="n">safe_key</span><span class="p">[:</span><span class="n">MAX_POD_ID_LEN</span> <span class="o">-</span> <span class="nb">len</span><span class="p">(</span><span class="n">safe_uuid</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">]</span> <span class="o">+</span> <span class="s2">"-"</span> <span class="o">+</span> <span class="n">safe_uuid</span>
+
+ <span class="k">return</span> <span class="n">safe_pod_id</span>
+
+ <span class="nd">@staticmethod</span>
+ <span class="k">def</span> <span class="nf">_create_pod_id</span><span class="p">(</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">task_id</span><span class="p">):</span>
+ <span class="n">safe_dag_id</span> <span class="o">=</span> <span class="n">AirflowKubernetesScheduler</span><span class="o">.</span><span class="n">_strip_unsafe_kubernetes_special_chars</span><span class="p">(</span>
+ <span class="n">dag_id</span><span class="p">)</span>
+ <span class="n">safe_task_id</span> <span class="o">=</span> <span class="n">AirflowKubernetesScheduler</span><span class="o">.</span><span class="n">_strip_unsafe_kubernetes_special_chars</span><span class="p">(</span>
+ <span class="n">task_id</span><span class="p">)</span>
+ <span class="n">safe_uuid</span> <span class="o">=</span> <span class="n">AirflowKubernetesScheduler</span><span class="o">.</span><span class="n">_strip_unsafe_kubernetes_special_chars</span><span class="p">(</span>
+ <span class="n">uuid4</span><span class="p">()</span><span class="o">.</span><span class="n">hex</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">AirflowKubernetesScheduler</span><span class="o">.</span><span class="n">_make_safe_pod_id</span><span class="p">(</span><span class="n">safe_dag_id</span><span class="p">,</span> <span class="n">safe_task_id</span><span class="p">,</span>
+ <span class="n">safe_uuid</span><span class="p">)</span>
+
+ <span class="nd">@staticmethod</span>
+ <span class="k">def</span> <span class="nf">_label_safe_datestring_to_datetime</span><span class="p">(</span><span class="n">string</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Kubernetes doesn't permit ":" in labels. ISO datetime format uses ":" but not</span>
+<span class="sd"> "_", let's</span>
+<span class="sd"> replace ":" with "_"</span>
+
+<span class="sd"> :param string: string</span>
+<span class="sd"> :return: datetime.datetime object</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="n">parser</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">string</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'_plus_'</span><span class="p">,</span> <span class="s1">'+'</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"_"</span><sp [...]
+
+ <span class="nd">@staticmethod</span>
+ <span class="k">def</span> <span class="nf">_datetime_to_label_safe_datestring</span><span class="p">(</span><span class="n">datetime_obj</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Kubernetes doesn't like ":" in labels, since ISO datetime format uses ":" but</span>
+<span class="sd"> not "_" let's</span>
+<span class="sd"> replace ":" with "_"</span>
+<span class="sd"> :param datetime_obj: datetime.datetime object</span>
+<span class="sd"> :return: ISO-like string representing the datetime</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="n">datetime_obj</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">":"</span><span class="p">,</span> <span class="s2">"_"</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'+'</span><span class="p">,</span> [...]
+
+ <span class="k">def</span> <span class="nf">_labels_to_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">labels</span><span class="p">):</span>
+ <span class="n">try_num</span> <span class="o">=</span> <span class="mi">1</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">try_num</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">labels</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'try_number'</span><span class="p">,</span> <span class="s1">'1'</span><span class="p">))</span>
+ <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">"could not get try_number as an int: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">labels</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'try_number'</span><span class="p">,</span> <span cl [...]
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="k">return</span> <span class="p">(</span>
+ <span class="n">labels</span><span class="p">[</span><span class="s1">'dag_id'</span><span class="p">],</span> <span class="n">labels</span><span class="p">[</span><span class="s1">'task_id'</span><span class="p">],</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_label_safe_datestring_to_datetime</span><span class="p">(</span><span class="n">labels</span><span class="p">[</span><span class="s1">'execution_date'</span><span class="p">]),</span>
+ <span class="n">try_num</span><span class="p">,</span>
+ <span class="p">)</span>
+ <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+ <span class="s1">'Error while converting labels to key; labels: </span><span class="si">%s</span><span class="s1">; exception: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">labels</span><span class="p">,</span> <span class="n">e</span>
+ <span class="p">)</span>
+ <span class="k">return</span> <span class="kc">None</span>
+
+
+<div class="viewcode-block" id="KubernetesExecutor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.kubernetes_executor.KubernetesExecutor">[docs]</a><span class="k">class</span> <span class="nc">KubernetesExecutor</span><span class="p">(</span><span class="n">BaseExecutor</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span> <span class="o">=</span> <span class="n">KubeConfig</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_scheduler</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_client</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">worker_uuid</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">KubernetesExecutor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">parallelism</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span><span class="o">.</span><span class="n">parallelism</span><span class="p">)</span>
+
+<div class="viewcode-block" id="KubernetesExecutor.clear_not_launched_queued_tasks"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.kubernetes_executor.KubernetesExecutor.clear_not_launched_queued_tasks">[docs]</a> <span class="nd">@provide_session</span>
+ <span class="k">def</span> <span class="nf">clear_not_launched_queued_tasks</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> If the airflow scheduler restarts with pending "Queued" tasks, the tasks may or</span>
+<span class="sd"> may not</span>
+<span class="sd"> have been launched Thus, on starting up the scheduler let's check every</span>
+<span class="sd"> "Queued" task to</span>
+<span class="sd"> see if it has been launched (ie: if there is a corresponding pod on kubernetes)</span>
+
+<span class="sd"> If it has been launched then do nothing, otherwise reset the state to "None" so</span>
+<span class="sd"> the task</span>
+<span class="sd"> will be rescheduled</span>
+
+<span class="sd"> This will not be necessary in a future version of airflow in which there is</span>
+<span class="sd"> proper support</span>
+<span class="sd"> for State.LAUNCHED</span>
+<span class="sd"> """</span>
+ <span class="n">queued_tasks</span> <span class="o">=</span> <span class="n">session</span>\
+ <span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TaskInstance</span><span class="p">)</span>\
+ <span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">QUEUED</span><span class="p">)</span><span class="o">.</span><span class="n">all</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'When executor started up, found </span><span class="si">%s</span><span class="s1"> queued task instances'</span><span class="p">,</span>
+ <span class="nb">len</span><span class="p">(</span><span class="n">queued_tasks</span><span class="p">)</span>
+ <span class="p">)</span>
+
+ <span class="k">for</span> <span class="n">task</span> <span class="ow">in</span> <span class="n">queued_tasks</span><span class="p">:</span>
+ <span class="n">dict_string</span> <span class="o">=</span> <span class="s2">"dag_id=</span><span class="si">{}</span><span class="s2">,task_id=</span><span class="si">{}</span><span class="s2">,execution_date=</span><span class="si">{}</span><span class="s2">,airflow-worker=</span><span class="si">{}</span><span class="s2">"</span> \
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">task</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span>
+ <span class="n">AirflowKubernetesScheduler</span><span class="o">.</span><span class="n">_datetime_to_label_safe_datestring</span><span class="p">(</span>
+ <span class="n">task</span><span class="o">.</span><span class="n">execution_date</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_uuid</span><span class="p">)</span>
+ <span class="n">kwargs</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">label_selector</span><span class="o">=</span><span class="n">dict_string</span><span class="p">)</span>
+ <span class="n">pod_list</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">kube_client</span><span class="o">.</span><span class="n">list_namespaced_pod</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span><span class="o">.</span><span class="n">kube_namespace</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">pod_list</span><span class="o">.</span><span class="n">items</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'TaskInstance: </span><span class="si">%s</span><span class="s1"> found in queued state but was not launched, '</span>
+ <span class="s1">'rescheduling'</span><span class="p">,</span> <span class="n">task</span>
+ <span class="p">)</span>
+ <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TaskInstance</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
+ <span class="n">TaskInstance</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">task</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
+ <span class="n">TaskInstance</span><span class="o">.</span><span class="n">task_id</span> <span class="o">==</span> <span class="n">task</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span>
+ <span class="n">TaskInstance</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">==</span> <span class="n">task</span><span class="o">.</span><span class="n">execution_date</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">state</span><span class="p">:</span> <span class="n">State</span><span class="o">.</span><span class="n">NONE</span><span class="p">})</span></div>
+
+ <span class="k">def</span> <span class="nf">_inject_secrets</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">_create_or_update_secret</span><span class="p">(</span><span class="n">secret_name</span><span class="p">,</span> <span class="n">secret_path</span><span class="p">):</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">kube_client</span><span class="o">.</span><span class="n">create_namespaced_secret</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span><span class="o">.</span><span class="n">executor_namespace</span><span class="p">,</span> <span class="n">kubernetes</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">V1Secret</span><span class="p">(</span>
+ <span class="n">data</span><span class="o">=</span><span class="p">{</span>
+ <span class="s1">'key.json'</span><span class="p">:</span> <span class="n">base64</span><span class="o">.</span><span class="n">b64encode</span><span class="p">(</span><span class="nb">open</span><span class="p">(</span><span class="n">secret_path</span><span class="p">,</span> <span class="s1">'r'</span><span class="p">)</span><span class="o">.</span><span class="n">read</span><span class="p">())},</span>
+ <span class="n">metadata</span><span class="o">=</span><span class="n">kubernetes</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">V1ObjectMeta</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="n">secret_name</span><span class="p">)))</span>
+ <span class="k">except</span> <span class="n">ApiException</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">e</span><span class="o">.</span><span class="n">status</span> <span class="o">==</span> <span class="mi">409</span><span class="p">:</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">kube_client</span><span class="o">.</span><span class="n">replace_namespaced_secret</span><span class="p">(</span>
+ <span class="n">secret_name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span><span class="o">.</span><span class="n">executor_namespace</span><span class="p">,</span>
+ <span class="n">kubernetes</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">V1Secret</span><span class="p">(</span>
+ <span class="n">data</span><span class="o">=</span><span class="p">{</span><span class="s1">'key.json'</span><span class="p">:</span> <span class="n">base64</span><span class="o">.</span><span class="n">b64encode</span><span class="p">(</span>
+ <span class="nb">open</span><span class="p">(</span><span class="n">secret_path</span><span class="p">,</span> <span class="s1">'r'</span><span class="p">)</span><span class="o">.</span><span class="n">read</span><span class="p">())},</span>
+ <span class="n">metadata</span><span class="o">=</span><span class="n">kubernetes</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">V1ObjectMeta</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="n">secret_name</span><span class="p">)))</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span>
+ <span class="s1">'Exception while trying to inject secret. '</span>
+ <span class="s1">'Secret name: </span><span class="si">%s</span><span class="s1">, error details: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">secret_name</span><span class="p">,</span> <span class="n">e</span>
+ <span class="p">)</span>
+ <span class="k">raise</span>
+
+ <span class="c1"># For each GCP service account key, inject it as a secret in executor</span>
+ <span class="c1"># namespace with the specific secret name configured in the airflow.cfg.</span>
+ <span class="c1"># We let exceptions to pass through to users.</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span><span class="o">.</span><span class="n">gcp_service_account_keys</span><span class="p">:</span>
+ <span class="n">name_path_pair_list</span> <span class="o">=</span> <span class="p">[</span>
+ <span class="p">{</span><span class="s1">'name'</span><span class="p">:</span> <span class="n">account_spec</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'='</span><span class="p">)[</span><span class="mi">0</span><span class="p">],</span>
+ <span class="s1">'path'</span><span class="p">:</span> <span class="n">account_spec</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'='</span><span class="p">)[</span><span class="mi">1</span><span class="p">]}</span>
+ <span class="k">for</span> <span class="n">account_spec</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span><span class="o">.</span><span class="n">gcp_service_account_keys</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">','</span><span class="p">)]</span>
+ <span class="k">for</span> <span class="n">service_account</span> <span class="ow">in</span> <span class="n">name_path_pair_list</span><span class="p">:</span>
+ <span class="n">_create_or_update_secret</span><span class="p">(</span><span class="n">service_account</span><span class="p">[</span><span class="s1">'name'</span><span class="p">],</span> <span class="n">service_account</span><span class="p">[</span><span class="s1">'path'</span><span class="p">])</span>
+
+<div class="viewcode-block" id="KubernetesExecutor.start"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.kubernetes_executor.KubernetesExecutor.start">[docs]</a> <span class="k">def</span> <span class="nf">start</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Start Kubernetes executor'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">worker_uuid</span> <span class="o">=</span> <span class="n">KubeWorkerIdentifier</span><span class="o">.</span><span class="n">get_or_create_current_kube_worker_uuid</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'Start with worker_uuid: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_uuid</span><span class="p">)</span>
+ <span class="c1"># always need to reset resource version since we don't know</span>
+ <span class="c1"># when we last started, note for behavior below</span>
+ <span class="c1"># https://github.com/kubernetes-client/python/blob/master/kubernetes/docs</span>
+ <span class="c1"># /CoreV1Api.md#list_namespaced_pod</span>
+ <span class="n">KubeResourceVersion</span><span class="o">.</span><span class="n">reset_resource_version</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span> <span class="o">=</span> <span class="n">Queue</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="n">Queue</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_client</span> <span class="o">=</span> <span class="n">get_kube_client</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_scheduler</span> <span class="o">=</span> <span class="n">AirflowKubernetesScheduler</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_client</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_uuid</span>
+ <span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_inject_secrets</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">clear_not_launched_queued_tasks</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="KubernetesExecutor.execute_async"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.kubernetes_executor.KubernetesExecutor.execute_async">[docs]</a> <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><spa [...]
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'Add task </span><span class="si">%s</span><span class="s1"> with command </span><span class="si">%s</span><span class="s1"> with executor_config </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">executor_config</span>
+ <span class="p">)</span>
+ <span class="n">kube_executor_config</span> <span class="o">=</span> <span class="n">KubernetesExecutorConfig</span><span class="o">.</span><span class="n">from_dict</span><span class="p">(</span><span class="n">executor_config</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">kube_executor_config</span><span class="p">))</span></div>
+
+<div class="viewcode-block" id="KubernetesExecutor.sync"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.kubernetes_executor.KubernetesExecutor.sync">[docs]</a> <span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">running</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'self.running: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">running</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">queued_tasks</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'self.queued: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">queued_tasks</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_scheduler</span><span class="o">.</span><span class="n">sync</span><span class="p">()</span>
+
+ <span class="n">last_resource_version</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">empty</span><span class="p">():</span>
+ <span class="n">results</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
+ <span class="n">key</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">pod_id</span><span class="p">,</span> <span class="n">resource_version</span> <span class="o">=</span> <span class="n">results</span>
+ <span class="n">last_resource_version</span> <span class="o">=</span> <span class="n">resource_version</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Changing state of </span><span class="si">%s</span><span class="s1"> to </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">results</span><span class="p">,</span> <span class="n">state</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_change_state</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">pod_id</span><span class="p">)</span>
+
+ <span class="n">KubeResourceVersion</span><span class="o">.</span><span class="n">checkpoint_resource_version</span><span class="p">(</span><span class="n">last_resource_version</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">empty</span><span class="p">():</span>
+ <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">kube_executor_config</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_scheduler</span><span class="o">.</span><span class="n">run_next</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">kube_executor_config</span><span class="p">))</span></div>
+
+ <span class="k">def</span> <span class="nf">_change_state</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">pod_id</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">state</span> <span class="o">!=</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">kube_scheduler</span><span class="o">.</span><span class="n">delete_pod</span><span class="p">(</span><span class="n">pod_id</span><span class="p">)</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Deleted pod: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">key</span><span class="p">))</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">running</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
+ <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'Could not find key: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">key</span><span class="p">))</span>
+ <span class="k">pass</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">event_buffer</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">state</span>
+ <span class="p">(</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">ex_time</span><span class="p">,</span> <span class="n">try_number</span><span class="p">)</span> <span class="o">=</span> <span class="n">key</span>
+ <span class="k">with</span> <span class="n">create_session</span><span class="p">()</span> <span class="k">as</span> <span class="n">session</span><span class="p">:</span>
+ <span class="n">item</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TaskInstance</span><span class="p">)</span><span class="o">.</span><span class="n">filter_by</span><span class="p">(</span>
+ <span class="n">dag_id</span><span class="o">=</span><span class="n">dag_id</span><span class="p">,</span>
+ <span class="n">task_id</span><span class="o">=</span><span class="n">task_id</span><span class="p">,</span>
+ <span class="n">execution_date</span><span class="o">=</span><span class="n">ex_time</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">one</span><span class="p">()</span>
+ <span class="k">if</span> <span class="n">state</span><span class="p">:</span>
+ <span class="n">item</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">state</span>
+ <span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
+
+<div class="viewcode-block" id="KubernetesExecutor.end"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.kubernetes_executor.KubernetesExecutor.end">[docs]</a> <span class="k">def</span> <span class="nf">end</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Shutting down Kubernetes executor'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">join</span><span class="p">()</span></div></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+ <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/executors/mesos_executor.html b/1.10.2/_modules/airflow/contrib/executors/mesos_executor.html
new file mode 100644
index 0000000..ec4460f
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/executors/mesos_executor.html
@@ -0,0 +1,549 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.executors.mesos_executor — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+ <div class="version">
+ 1.10.2
+ </div>
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.executors.mesos_executor</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.executors.mesos_executor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">future</span> <span class="k">import</span> <span class="n">standard_library</span>
+
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+<span class="kn">from</span> <span class="nn">airflow.www.utils</span> <span class="k">import</span> <span class="n">LoginMixin</span>
+
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
+<span class="kn">from</span> <span class="nn">queue</span> <span class="k">import</span> <span class="n">Queue</span>
+
+<span class="kn">import</span> <span class="nn">mesos.interface</span>
+<span class="kn">from</span> <span class="nn">mesos.interface</span> <span class="k">import</span> <span class="n">mesos_pb2</span>
+<span class="kn">import</span> <span class="nn">mesos.native</span>
+
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">configuration</span>
+<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="k">import</span> <span class="n">BaseExecutor</span>
+<span class="kn">from</span> <span class="nn">airflow.settings</span> <span class="k">import</span> <span class="n">Session</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+
+<span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
+<span class="n">DEFAULT_FRAMEWORK_NAME</span> <span class="o">=</span> <span class="s1">'Airflow'</span>
+<span class="n">FRAMEWORK_CONNID_PREFIX</span> <span class="o">=</span> <span class="s1">'mesos_framework_'</span>
+
+
+<span class="k">def</span> <span class="nf">get_framework_name</span><span class="p">():</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FRAMEWORK_NAME'</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">DEFAULT_FRAMEWORK_NAME</span>
+ <span class="k">return</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FRAMEWORK_NAME'</span><span class="p">)</span>
+
+
+<span class="c1"># AirflowMesosScheduler, implements Mesos Scheduler interface</span>
+<span class="c1"># To schedule airflow jobs on mesos</span>
+<span class="k">class</span> <span class="nc">AirflowMesosScheduler</span><span class="p">(</span><span class="n">mesos</span><span class="o">.</span><span class="n">interface</span><span class="o">.</span><span class="n">Scheduler</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Airflow Mesos scheduler implements mesos scheduler interface</span>
+<span class="sd"> to schedule airflow tasks on mesos.</span>
+<span class="sd"> Basically, it schedules a command like</span>
+<span class="sd"> 'airflow run <dag_id> <task_instance_id> <start_date> --local -p=<pickle>'</span>
+<span class="sd"> to run on a mesos slave.</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">task_queue</span><span class="p">,</span>
+ <span class="n">result_queue</span><span class="p">,</span>
+ <span class="n">task_cpu</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
+ <span class="n">task_mem</span><span class="o">=</span><span class="mi">256</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span> <span class="o">=</span> <span class="n">task_queue</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="n">result_queue</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span> <span class="o">=</span> <span class="n">task_cpu</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span> <span class="o">=</span> <span class="n">task_mem</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">task_counter</span> <span class="o">=</span> <span class="mi">0</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DOCKER_IMAGE_SLAVE'</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mesos_slave_docker_image</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DOCKER_IMAGE_SLAVE'</span>
+ <span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">registered</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">frameworkId</span><span class="p">,</span> <span class="n">masterInfo</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler registered to Mesos with framework ID </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
+ <span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">)</span> <span class="ow">and</span> \
+ <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span>
+ <span class="c1"># Import here to work around a circular import error</span>
+ <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">Connection</span>
+
+ <span class="c1"># Update the Framework ID in the database.</span>
+ <span class="n">session</span> <span class="o">=</span> <span class="n">Session</span><span class="p">()</span>
+ <span class="n">conn_id</span> <span class="o">=</span> <span class="n">FRAMEWORK_CONNID_PREFIX</span> <span class="o">+</span> <span class="n">get_framework_name</span><span class="p">()</span>
+ <span class="n">connection</span> <span class="o">=</span> <span class="n">Session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">Connection</span><span class="p">)</span><span class="o">.</span><span class="n">filter_by</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
+ <span class="k">if</span> <span class="n">connection</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">connection</span> <span class="o">=</span> <span class="n">Connection</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">,</span> <span class="n">conn_type</span><span class="o">=</span><span class="s1">'mesos_framework-id'</span><span class="p">,</span>
+ <span class="n">extra</span><span class="o">=</span><span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">connection</span><span class="o">.</span><span class="n">extra</span> <span class="o">=</span> <span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span>
+
+ <span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">connection</span><span class="p">)</span>
+ <span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
+ <span class="n">Session</span><span class="o">.</span><span class="n">remove</span><span class="p">()</span>
+
+ <span class="k">def</span> <span class="nf">reregistered</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">masterInfo</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler re-registered to mesos"</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">disconnected</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler disconnected from mesos"</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">offerRescinded</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">offerId</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler offer </span><span class="si">%s</span><span class="s2"> rescinded"</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">offerId</span><span class="p">))</span>
+
+ <span class="k">def</span> <span class="nf">frameworkMessage</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">executorId</span><span class="p">,</span> <span class="n">slaveId</span><span class="p">,</span> <span class="n">message</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler received framework message </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">executorLost</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">executorId</span><span class="p">,</span> <span class="n">slaveId</span><span class="p">,</span> <span class="n">status</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"AirflowScheduler executor </span><span class="si">%s</span><span class="s2"> lost"</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">executorId</span><span class="p">))</span>
+
+ <span class="k">def</span> <span class="nf">slaveLost</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">slaveId</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"AirflowScheduler slave </span><span class="si">%s</span><span class="s2"> lost"</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">slaveId</span><span class="p">))</span>
+
+ <span class="k">def</span> <span class="nf">error</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">message</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"AirflowScheduler driver aborted </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"AirflowScheduler driver aborted </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">message</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">resourceOffers</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">offers</span><span class="p">):</span>
+ <span class="k">for</span> <span class="n">offer</span> <span class="ow">in</span> <span class="n">offers</span><span class="p">:</span>
+ <span class="n">tasks</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="n">offerCpus</span> <span class="o">=</span> <span class="mi">0</span>
+ <span class="n">offerMem</span> <span class="o">=</span> <span class="mi">0</span>
+ <span class="k">for</span> <span class="n">resource</span> <span class="ow">in</span> <span class="n">offer</span><span class="o">.</span><span class="n">resources</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">resource</span><span class="o">.</span><span class="n">name</span> <span class="o">==</span> <span class="s2">"cpus"</span><span class="p">:</span>
+ <span class="n">offerCpus</span> <span class="o">+=</span> <span class="n">resource</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span>
+ <span class="k">elif</span> <span class="n">resource</span><span class="o">.</span><span class="n">name</span> <span class="o">==</span> <span class="s2">"mem"</span><span class="p">:</span>
+ <span class="n">offerMem</span> <span class="o">+=</span> <span class="n">resource</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Received offer </span><span class="si">%s</span><span class="s2"> with cpus: </span><span class="si">%s</span><span class="s2"> and mem: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
+ <span class="n">offer</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">value</span><span class="p">,</span> <span class="n">offerCpus</span><span class="p">,</span> <span class="n">offerMem</span><span class="p">)</span>
+
+ <span class="n">remainingCpus</span> <span class="o">=</span> <span class="n">offerCpus</span>
+ <span class="n">remainingMem</span> <span class="o">=</span> <span class="n">offerMem</span>
+
+ <span class="k">while</span> <span class="p">(</span><span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">empty</span><span class="p">())</span> <span class="ow">and</span> \
+ <span class="n">remainingCpus</span> <span class="o">>=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span> <span class="ow">and</span> \
+ <span class="n">remainingMem</span> <span class="o">>=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span><span class="p">:</span>
+ <span class="n">key</span><span class="p">,</span> <span class="n">cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
+ <span class="n">tid</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_counter</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">task_counter</span> <span class="o">+=</span> <span class="mi">1</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span><span class="p">[</span><span class="nb">str</span><span class="p">(</span><span class="n">tid</span><span class="p">)]</span> <span class="o">=</span> <span class="n">key</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Launching task </span><span class="si">%d</span><span class="s2"> using offer </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">tid</span><span class="p">,</span> <span class="n">offer</span><span class="o">.</span><span class="n">id</span><span [...]
+
+ <span class="n">task</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TaskInfo</span><span class="p">()</span>
+ <span class="n">task</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">tid</span><span class="p">)</span>
+ <span class="n">task</span><span class="o">.</span><span class="n">slave_id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">offer</span><span class="o">.</span><span class="n">slave_id</span><span class="o">.</span><span class="n">value</span>
+ <span class="n">task</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="s2">"AirflowTask </span><span class="si">%d</span><span class="s2">"</span> <span class="o">%</span> <span class="n">tid</span>
+
+ <span class="n">cpus</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">add</span><span class="p">()</span>
+ <span class="n">cpus</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="s2">"cpus"</span>
+ <span class="n">cpus</span><span class="o">.</span><span class="n">type</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">Value</span><span class="o">.</span><span class="n">SCALAR</span>
+ <span class="n">cpus</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span>
+
+ <span class="n">mem</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">add</span><span class="p">()</span>
+ <span class="n">mem</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="s2">"mem"</span>
+ <span class="n">mem</span><span class="o">.</span><span class="n">type</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">Value</span><span class="o">.</span><span class="n">SCALAR</span>
+ <span class="n">mem</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span>
+
+ <span class="n">command</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">CommandInfo</span><span class="p">()</span>
+ <span class="n">command</span><span class="o">.</span><span class="n">shell</span> <span class="o">=</span> <span class="kc">True</span>
+ <span class="n">command</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">cmd</span>
+ <span class="n">task</span><span class="o">.</span><span class="n">command</span><span class="o">.</span><span class="n">MergeFrom</span><span class="p">(</span><span class="n">command</span><span class="p">)</span>
+
+ <span class="c1"># If docker image for airflow is specified in config then pull that</span>
+ <span class="c1"># image before running the above airflow command</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mesos_slave_docker_image</span><span class="p">:</span>
+ <span class="n">network</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="o">.</span><span class="n">DockerInfo</span><span class="o">.</span><span class="n">Network</span><span class="o">.</span><span class="n">Value</span><span class="p">(</span><span class="s1">'BRIDGE'</span><span class="p">)</span>
+ <span class="n">docker</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="o">.</span><span class="n">DockerInfo</span><span class="p">(</span>
+ <span class="n">image</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mesos_slave_docker_image</span><span class="p">,</span>
+ <span class="n">force_pull_image</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">network</span><span class="o">=</span><span class="n">network</span>
+ <span class="p">)</span>
+ <span class="n">container</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="p">(</span>
+ <span class="nb">type</span><span class="o">=</span><span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="o">.</span><span class="n">DOCKER</span><span class="p">,</span>
+ <span class="n">docker</span><span class="o">=</span><span class="n">docker</span>
+ <span class="p">)</span>
+ <span class="n">task</span><span class="o">.</span><span class="n">container</span><span class="o">.</span><span class="n">MergeFrom</span><span class="p">(</span><span class="n">container</span><span class="p">)</span>
+
+ <span class="n">tasks</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">task</span><span class="p">)</span>
+
+ <span class="n">remainingCpus</span> <span class="o">-=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span>
+ <span class="n">remainingMem</span> <span class="o">-=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span>
+
+ <span class="n">driver</span><span class="o">.</span><span class="n">launchTasks</span><span class="p">(</span><span class="n">offer</span><span class="o">.</span><span class="n">id</span><span class="p">,</span> <span class="n">tasks</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">statusUpdate</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">update</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s2">"Task </span><span class="si">%s</span><span class="s2"> is in state </span><span class="si">%s</span><span class="s2">, data </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
+ <span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">,</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TaskState</span><span class="o">.</span><span class="n">Name</span><span class="p">(</span><span class="n">update</span><span class="o">.</span><span class="n">state</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span [...]
+ <span class="p">)</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">key</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span><span class="p">[</span><span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">]</span>
+ <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
+ <span class="c1"># The map may not contain an item if the framework re-registered</span>
+ <span class="c1"># after a failover.</span>
+ <span class="c1"># Discard these tasks.</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Unrecognised task key </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
+ <span class="k">return</span>
+
+ <span class="k">if</span> <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_FINISHED</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">))</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">task_done</span><span class="p">()</span>
+
+ <span class="k">if</span> <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_LOST</span> <span class="ow">or</span> \
+ <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_KILLED</span> <span class="ow">or</span> \
+ <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_FAILED</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">FAILED</span><span class="p">))</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">task_done</span><span class="p">()</span>
+
+
+<div class="viewcode-block" id="MesosExecutor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor">[docs]</a><span class="k">class</span> <span class="nc">MesosExecutor</span><span class="p">(</span><span class="n">BaseExecutor</span><span class="p">,</span> <span class="n">LoginMixin</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> MesosExecutor allows distributing the execution of task</span>
+<span class="sd"> instances to multiple mesos workers.</span>
+
+<span class="sd"> Apache Mesos is a distributed systems kernel which abstracts</span>
+<span class="sd"> CPU, memory, storage, and other compute resources away from</span>
+<span class="sd"> machines (physical or virtual), enabling fault-tolerant and</span>
+<span class="sd"> elastic distributed systems to easily be built and run effectively.</span>
+<span class="sd"> See http://mesos.apache.org/</span>
+<span class="sd"> """</span>
+<div class="viewcode-block" id="MesosExecutor.start"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor.start">[docs]</a> <span class="k">def</span> <span class="nf">start</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span> <span class="o">=</span> <span class="n">Queue</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="n">Queue</span><span class="p">()</span>
+ <span class="n">framework</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">FrameworkInfo</span><span class="p">()</span>
+ <span class="n">framework</span><span class="o">.</span><span class="n">user</span> <span class="o">=</span> <span class="s1">''</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'MASTER'</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Expecting mesos master URL for mesos executor"</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"mesos.master not provided for mesos executor"</span><span class="p">)</span>
+
+ <span class="n">master</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'MASTER'</span><span class="p">)</span>
+
+ <span class="n">framework</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">get_framework_name</span><span class="p">()</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_CPU'</span><span class="p">):</span>
+ <span class="n">task_cpu</span> <span class="o">=</span> <span class="mi">1</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">task_cpu</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_CPU'</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_MEMORY'</span><span class="p">):</span>
+ <span class="n">task_memory</span> <span class="o">=</span> <span class="mi">256</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">task_memory</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_MEMORY'</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">):</span>
+ <span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="kc">True</span>
+
+ <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span>
+ <span class="c1"># Import here to work around a circular import error</span>
+ <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">Connection</span>
+
+ <span class="c1"># Query the database to get the ID of the Mesos Framework, if available.</span>
+ <span class="n">conn_id</span> <span class="o">=</span> <span class="n">FRAMEWORK_CONNID_PREFIX</span> <span class="o">+</span> <span class="n">framework</span><span class="o">.</span><span class="n">name</span>
+ <span class="n">session</span> <span class="o">=</span> <span class="n">Session</span><span class="p">()</span>
+ <span class="n">connection</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">Connection</span><span class="p">)</span><span class="o">.</span><span class="n">filter_by</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p [...]
+ <span class="k">if</span> <span class="n">connection</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="c1"># Set the Framework ID to let the scheduler reconnect</span>
+ <span class="c1"># with running tasks.</span>
+ <span class="n">framework</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">connection</span><span class="o">.</span><span class="n">extra</span>
+
+ <span class="n">framework</span><span class="o">.</span><span class="n">failover_timeout</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span>
+ <span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span>
+ <span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="kc">False</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'MesosFramework master : </span><span class="si">%s</span><span class="s1">, name : </span><span class="si">%s</span><span class="s1">, cpu : </span><span class="si">%s</span><span class="s1">, mem : </span><span class="si">%s</span><span class="s1">, checkpoint : </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">master</span><span class="p">,</span> <span class="n">framework</span><span class="o">.</span><span class="n">name</span><span class="p">,</span>
+ <span class="nb">str</span><span class="p">(</span><span class="n">task_cpu</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">task_memory</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span><span class="p">)</span>
+ <span class="p">)</span>
+
+ <span class="n">implicit_acknowledgements</span> <span class="o">=</span> <span class="mi">1</span>
+
+ <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'AUTHENTICATE'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_PRINCIPAL'</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Expecting authentication principal in the environment"</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s2">"mesos.default_principal not provided in authenticated mode"</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_SECRET'</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Expecting authentication secret in the environment"</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s2">"mesos.default_secret not provided in authenticated mode"</span><span class="p">)</span>
+
+ <span class="n">credential</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">Credential</span><span class="p">()</span>
+ <span class="n">credential</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_PRINCIPAL'</span><span class="p">)</span>
+ <span class="n">credential</span><span class="o">.</span><span class="n">secret</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_SECRET'</span><span class="p">)</span>
+
+ <span class="n">framework</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="n">credential</span><span class="o">.</span><span class="n">principal</span>
+
+ <span class="n">driver</span> <span class="o">=</span> <span class="n">mesos</span><span class="o">.</span><span class="n">native</span><span class="o">.</span><span class="n">MesosSchedulerDriver</span><span class="p">(</span>
+ <span class="n">AirflowMesosScheduler</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="p">,</span>
+ <span class="n">task_cpu</span><span class="p">,</span>
+ <span class="n">task_memory</span><span class="p">),</span>
+ <span class="n">framework</span><span class="p">,</span>
+ <span class="n">master</span><span class="p">,</span>
+ <span class="n">implicit_acknowledgements</span><span class="p">,</span>
+ <span class="n">credential</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">framework</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="s1">'Airflow'</span>
+ <span class="n">driver</span> <span class="o">=</span> <span class="n">mesos</span><span class="o">.</span><span class="n">native</span><span class="o">.</span><span class="n">MesosSchedulerDriver</span><span class="p">(</span>
+ <span class="n">AirflowMesosScheduler</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="p">,</span>
+ <span class="n">task_cpu</span><span class="p">,</span>
+ <span class="n">task_memory</span><span class="p">),</span>
+ <span class="n">framework</span><span class="p">,</span>
+ <span class="n">master</span><span class="p">,</span>
+ <span class="n">implicit_acknowledgements</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span> <span class="o">=</span> <span class="n">driver</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span><span class="o">.</span><span class="n">start</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="MesosExecutor.execute_async"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor.execute_async">[docs]</a> <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</ [...]
+ <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">))</span></div>
+
+<div class="viewcode-block" id="MesosExecutor.sync"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor.sync">[docs]</a> <span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">empty</span><span class="p">():</span>
+ <span class="n">results</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">change_state</span><span class="p">(</span><span class="o">*</span><span class="n">results</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="MesosExecutor.end"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor.end">[docs]</a> <span class="k">def</span> <span class="nf">end</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span><span class="o">.</span><span class="n">stop</span><span class="p">()</span></div></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+ <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/aws_athena_hook.html b/1.10.2/_modules/airflow/contrib/hooks/aws_athena_hook.html
new file mode 100644
index 0000000..a310379
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/aws_athena_hook.html
@@ -0,0 +1,369 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.aws_athena_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+ <div class="version">
+ 1.10.2
+ </div>
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.aws_athena_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.aws_athena_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">time</span> <span class="k">import</span> <span class="n">sleep</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.aws_hook</span> <span class="k">import</span> <span class="n">AwsHook</span>
+
+
+<div class="viewcode-block" id="AWSAthenaHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_athena_hook.AWSAthenaHook">[docs]</a><span class="k">class</span> <span class="nc">AWSAthenaHook</span><span class="p">(</span><span class="n">AwsHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with AWS Athena to run, poll queries and return query results</span>
+
+<span class="sd"> :param aws_conn_id: aws connection to use.</span>
+<span class="sd"> :type aws_conn_id: str</span>
+<span class="sd"> :param sleep_time: Time to wait between two consecutive call to check query status on athena</span>
+<span class="sd"> :type sleep_time: int</span>
+<span class="sd"> """</span>
+
+ <span class="n">INTERMEDIATE_STATES</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'QUEUED'</span><span class="p">,</span> <span class="s1">'RUNNING'</span><span class="p">,)</span>
+ <span class="n">FAILURE_STATES</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'FAILED'</span><span class="p">,</span> <span class="s1">'CANCELLED'</span><span class="p">,)</span>
+ <span class="n">SUCCESS_STATES</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'SUCCEEDED'</span><span class="p">,)</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">aws_conn_id</span><span class="o">=</span><span class="s1">'aws_default'</span><span class="p">,</span> <span class="n">sleep_time</span><span class="o">=</span><span class="mi">30</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n" [...]
+ <span class="nb">super</span><span class="p">(</span><span class="n">AWSAthenaHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sleep_time</span> <span class="o">=</span> <span class="n">sleep_time</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="kc">None</span>
+
+<div class="viewcode-block" id="AWSAthenaHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_athena_hook.AWSAthenaHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> check if aws conn exists already or create one and return it</span>
+
+<span class="sd"> :return: boto3 session</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_client_type</span><span class="p">(</span><span class="s1">'athena'</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span></div>
+
+<div class="viewcode-block" id="AWSAthenaHook.run_query"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_athena_hook.AWSAthenaHook.run_query">[docs]</a> <span class="k">def</span> <span class="nf">run_query</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query</span><span class="p">,</span> <span class="n">query_context</span><span class="p">,</span> <span class="n">result_configuration</span><span class [...]
+ <span class="sd">"""</span>
+<span class="sd"> Run Presto query on athena with provided config and return submitted query_execution_id</span>
+
+<span class="sd"> :param query: Presto query to run</span>
+<span class="sd"> :type query: str</span>
+<span class="sd"> :param query_context: Context in which query need to be run</span>
+<span class="sd"> :type query_context: dict</span>
+<span class="sd"> :param result_configuration: Dict with path to store results in and config related to encryption</span>
+<span class="sd"> :type result_configuration: dict</span>
+<span class="sd"> :param client_request_token: Unique token created by user to avoid multiple executions of same query</span>
+<span class="sd"> :type client_request_token: str</span>
+<span class="sd"> :return: str</span>
+<span class="sd"> """</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">start_query_execution</span><span class="p">(</span><span class="n">QueryString</span><span class="o">=</span><span class="n">query</span><span class="p">,</span>
+ <span class="n">ClientRequestToken</span><span class="o">=</span><span class="n">client_request_token</span><span class="p">,</span>
+ <span class="n">QueryExecutionContext</span><span class="o">=</span><span class="n">query_context</span><span class="p">,</span>
+ <span class="n">ResultConfiguration</span><span class="o">=</span><span class="n">result_configuration</span><span class="p">)</span>
+ <span class="n">query_execution_id</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">'QueryExecutionId'</span><span class="p">]</span>
+ <span class="k">return</span> <span class="n">query_execution_id</span></div>
+
+<div class="viewcode-block" id="AWSAthenaHook.check_query_status"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_athena_hook.AWSAthenaHook.check_query_status">[docs]</a> <span class="k">def</span> <span class="nf">check_query_status</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query_execution_id</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Fetch the status of submitted athena query. Returns None or one of valid query states.</span>
+
+<span class="sd"> :param query_execution_id: Id of submitted athena query</span>
+<span class="sd"> :type query_execution_id: str</span>
+<span class="sd"> :return: str</span>
+<span class="sd"> """</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">get_query_execution</span><span class="p">(</span><span class="n">QueryExecutionId</span><span class="o">=</span><span class="n">query_execution_id</span><span class="p">)</span>
+ <span class="n">state</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">state</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">'QueryExecution'</span><span class="p">][</span><span class="s1">'Status'</span><span class="p">][</span><span class="s1">'State'</span><span class="p">]</span>
+ <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">ex</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">'Exception while getting query state'</span><span class="p">,</span> <span class="n">ex</span><span class="p">)</span>
+ <span class="k">finally</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">state</span></div>
+
+<div class="viewcode-block" id="AWSAthenaHook.get_query_results"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_athena_hook.AWSAthenaHook.get_query_results">[docs]</a> <span class="k">def</span> <span class="nf">get_query_results</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query_execution_id</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Fetch submitted athena query results. returns none if query is in intermediate state or</span>
+<span class="sd"> failed/cancelled state else dict of query output</span>
+
+<span class="sd"> :param query_execution_id: Id of submitted athena query</span>
+<span class="sd"> :type query_execution_id: str</span>
+<span class="sd"> :return: dict</span>
+<span class="sd"> """</span>
+ <span class="n">query_state</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">check_query_status</span><span class="p">(</span><span class="n">query_execution_id</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">query_state</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">'Invalid Query state'</span><span class="p">)</span>
+ <span class="k">return</span> <span class="kc">None</span>
+ <span class="k">elif</span> <span class="n">query_state</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">INTERMEDIATE_STATES</span> <span class="ow">or</span> <span class="n">query_state</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">FAILURE_STATES</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">'Query is in </span><span class="si">{state}</span><span class="s1"> state. Cannot fetch results'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">state</span><span class="o">=</span><span class="n">query_state</span><span class="p">))</span>
+ <span class="k">return</span> <span class="kc">None</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">get_query_results</span><span class="p">(</span><span class="n">QueryExecutionId</span><span class="o">=</span><span class="n">query_execution_id</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AWSAthenaHook.poll_query_status"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_athena_hook.AWSAthenaHook.poll_query_status">[docs]</a> <span class="k">def</span> <span class="nf">poll_query_status</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query_execution_id</span><span class="p">,</span> <span class="n">max_tries</span><span class="o">=</span><span class="kc">None< [...]
+ <span class="sd">"""</span>
+<span class="sd"> Poll the status of submitted athena query until query state reaches final state.</span>
+<span class="sd"> Returns one of the final states</span>
+
+<span class="sd"> :param query_execution_id: Id of submitted athena query</span>
+<span class="sd"> :type query_execution_id: str</span>
+<span class="sd"> :param max_tries: Number of times to poll for query state before function exits</span>
+<span class="sd"> :type max_tries: int</span>
+<span class="sd"> :return: str</span>
+<span class="sd"> """</span>
+ <span class="n">try_number</span> <span class="o">=</span> <span class="mi">1</span>
+ <span class="n">final_query_state</span> <span class="o">=</span> <span class="kc">None</span> <span class="c1"># Query state when query reaches final state or max_tries reached</span>
+ <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+ <span class="n">query_state</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">check_query_status</span><span class="p">(</span><span class="n">query_execution_id</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">query_state</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Trial </span><span class="si">{try_number}</span><span class="s1">: Invalid query state. Retrying again'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">try_number</span><span class="o">=</span><span class="n">try_number</span><span class="p">))</span>
+ <span class="k">elif</span> <span class="n">query_state</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">INTERMEDIATE_STATES</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Trial </span><span class="si">{try_number}</span><span class="s1">: Query is still in an intermediate state - </span><span class="si">{state}</span><span class="s1">'</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">try_number</span><span class="o">=</span><span class="n">try_number</span><span class="p">,</span> <span class="n">state</span><span class="o">=</span><span class="n">query_state</span><span class="p">))</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Trial </span><span class="si">{try_number}</span><span class="s1">: Query execution completed. Final state is </span><span class="si">{state}</span><span class="s1">'</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">try_number</span><span class="o">=</span><span class="n">try_number</span><span class="p">,</span> <span class="n">state</span><span class="o">=</span><span class="n">query_state</span><span class="p">))</span>
+ <span class="n">final_query_state</span> <span class="o">=</span> <span class="n">query_state</span>
+ <span class="k">break</span>
+ <span class="k">if</span> <span class="n">max_tries</span> <span class="ow">and</span> <span class="n">try_number</span> <span class="o">>=</span> <span class="n">max_tries</span><span class="p">:</span> <span class="c1"># Break loop if max_tries reached</span>
+ <span class="n">final_query_state</span> <span class="o">=</span> <span class="n">query_state</span>
+ <span class="k">break</span>
+ <span class="n">try_number</span> <span class="o">+=</span> <span class="mi">1</span>
+ <span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sleep_time</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">final_query_state</span></div>
+
+<div class="viewcode-block" id="AWSAthenaHook.stop_query"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_athena_hook.AWSAthenaHook.stop_query">[docs]</a> <span class="k">def</span> <span class="nf">stop_query</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query_execution_id</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Cancel the submitted athena query</span>
+
+<span class="sd"> :param query_execution_id: Id of submitted athena query</span>
+<span class="sd"> :type query_execution_id: str</span>
+<span class="sd"> :return: dict</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">stop_query_execution</span><span class="p">(</span><span class="n">QueryExecutionId</span><span class="o">=</span><span class="n">query_execution_id</span><span class="p">)</span></div></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+ <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html b/1.10.2/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html
new file mode 100644
index 0000000..3563f0d
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html
@@ -0,0 +1,288 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.aws_dynamodb_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+ <div class="version">
+ 1.10.2
+ </div>
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.aws_dynamodb_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.aws_dynamodb_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.aws_hook</span> <span class="k">import</span> <span class="n">AwsHook</span>
+
+
+<div class="viewcode-block" id="AwsDynamoDBHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_dynamodb_hook.AwsDynamoDBHook">[docs]</a><span class="k">class</span> <span class="nc">AwsDynamoDBHook</span><span class="p">(</span><span class="n">AwsHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with AWS DynamoDB.</span>
+
+<span class="sd"> :param table_keys: partition key and sort key</span>
+<span class="sd"> :type table_keys: list</span>
+<span class="sd"> :param table_name: target DynamoDB table</span>
+<span class="sd"> :type table_name: str</span>
+<span class="sd"> :param region_name: aws region name (example: us-east-1)</span>
+<span class="sd"> :type region_name: str</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">table_keys</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">table_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">table_keys</span> <span class="o">=</span> <span class="n">table_keys</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">table_name</span> <span class="o">=</span> <span class="n">table_name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">region_name</span> <span class="o">=</span> <span class="n">region_name</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">AwsDynamoDBHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_resource_type</span><span class="p">(</span><span class="s1">'dynamodb'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">region_name</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span>
+
+<div class="viewcode-block" id="AwsDynamoDBHook.write_batch_data"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_dynamodb_hook.AwsDynamoDBHook.write_batch_data">[docs]</a> <span class="k">def</span> <span class="nf">write_batch_data</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">items</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Write batch items to dynamodb table with provisioned throughout capacity.</span>
+<span class="sd"> """</span>
+
+ <span class="n">dynamodb_conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">table</span> <span class="o">=</span> <span class="n">dynamodb_conn</span><span class="o">.</span><span class="n">Table</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">table_name</span><span class="p">)</span>
+
+ <span class="k">with</span> <span class="n">table</span><span class="o">.</span><span class="n">batch_writer</span><span class="p">(</span><span class="n">overwrite_by_pkeys</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_keys</span><span class="p">)</span> <span class="k">as</span> <span class="n">batch</span><span class="p">:</span>
+ <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">items</span><span class="p">:</span>
+ <span class="n">batch</span><span class="o">.</span><span class="n">put_item</span><span class="p">(</span><span class="n">Item</span><span class="o">=</span><span class="n">item</span><span class="p">)</span>
+ <span class="k">return</span> <span class="kc">True</span>
+ <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">general_error</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'Failed to insert items in dynamodb, error: </span><span class="si">{error}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">error</span><span class="o">=</span><span class="nb">str</span><span class="p">(</span><span class="n">general_error</span><span class="p">)</span>
+ <span class="p">)</span>
+ <span class="p">)</span></div></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+ <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/aws_firehose_hook.html b/1.10.2/_modules/airflow/contrib/hooks/aws_firehose_hook.html
new file mode 100644
index 0000000..feb126a
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/aws_firehose_hook.html
@@ -0,0 +1,275 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.aws_firehose_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+ <div class="version">
+ 1.10.2
+ </div>
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.aws_firehose_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.aws_firehose_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.aws_hook</span> <span class="k">import</span> <span class="n">AwsHook</span>
+
+
+<div class="viewcode-block" id="AwsFirehoseHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_firehose_hook.AwsFirehoseHook">[docs]</a><span class="k">class</span> <span class="nc">AwsFirehoseHook</span><span class="p">(</span><span class="n">AwsHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with AWS Kinesis Firehose.</span>
+<span class="sd"> :param delivery_stream: Name of the delivery stream</span>
+<span class="sd"> :type delivery_stream: str</span>
+<span class="sd"> :param region_name: AWS region name (example: us-east-1)</span>
+<span class="sd"> :type region_name: str</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">delivery_stream</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">delivery_stream</span> <span class="o">=</span> <span class="n">delivery_stream</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">region_name</span> <span class="o">=</span> <span class="n">region_name</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">AwsFirehoseHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+<div class="viewcode-block" id="AwsFirehoseHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_firehose_hook.AwsFirehoseHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns AwsHook connection object.</span>
+<span class="sd"> """</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_client_type</span><span class="p">(</span><span class="s1">'firehose'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">region_name</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span></div>
+
+<div class="viewcode-block" id="AwsFirehoseHook.put_records"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_firehose_hook.AwsFirehoseHook.put_records">[docs]</a> <span class="k">def</span> <span class="nf">put_records</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">records</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Write batch records to Kinesis Firehose</span>
+<span class="sd"> """</span>
+
+ <span class="n">firehose_conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+ <span class="n">response</span> <span class="o">=</span> <span class="n">firehose_conn</span><span class="o">.</span><span class="n">put_record_batch</span><span class="p">(</span>
+ <span class="n">DeliveryStreamName</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delivery_stream</span><span class="p">,</span>
+ <span class="n">Records</span><span class="o">=</span><span class="n">records</span>
+ <span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">response</span></div></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+ <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/aws_glue_catalog_hook.html b/1.10.2/_modules/airflow/contrib/hooks/aws_glue_catalog_hook.html
new file mode 100644
index 0000000..e2403e0
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/aws_glue_catalog_hook.html
@@ -0,0 +1,337 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.aws_glue_catalog_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+ <div class="version">
+ 1.10.2
+ </div>
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.aws_glue_catalog_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.aws_glue_catalog_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.aws_hook</span> <span class="k">import</span> <span class="n">AwsHook</span>
+
+
+<div class="viewcode-block" id="AwsGlueCatalogHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_glue_catalog_hook.AwsGlueCatalogHook">[docs]</a><span class="k">class</span> <span class="nc">AwsGlueCatalogHook</span><span class="p">(</span><span class="n">AwsHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with AWS Glue Catalog</span>
+
+<span class="sd"> :param aws_conn_id: ID of the Airflow connection where</span>
+<span class="sd"> credentials and extra configuration are stored</span>
+<span class="sd"> :type aws_conn_id: str</span>
+<span class="sd"> :param region_name: aws region name (example: us-east-1)</span>
+<span class="sd"> :type region_name: str</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">aws_conn_id</span><span class="o">=</span><span class="s1">'aws_default'</span><span class="p">,</span>
+ <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">region_name</span> <span class="o">=</span> <span class="n">region_name</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">AwsGlueCatalogHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="n">aws_conn_id</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n" [...]
+
+<div class="viewcode-block" id="AwsGlueCatalogHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_glue_catalog_hook.AwsGlueCatalogHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns glue connection object.</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_client_type</span><span class="p">(</span><span class="s1">'glue'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">region_name</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span></div>
+
+<div class="viewcode-block" id="AwsGlueCatalogHook.get_partitions"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_glue_catalog_hook.AwsGlueCatalogHook.get_partitions">[docs]</a> <span class="k">def</span> <span class="nf">get_partitions</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">database_name</span><span class="p">,</span>
+ <span class="n">table_name</span><span class="p">,</span>
+ <span class="n">expression</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span>
+ <span class="n">page_size</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">max_items</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Retrieves the partition values for a table.</span>
+
+<span class="sd"> :param database_name: The name of the catalog database where the partitions reside.</span>
+<span class="sd"> :type database_name: str</span>
+<span class="sd"> :param table_name: The name of the partitions' table.</span>
+<span class="sd"> :type table_name: str</span>
+<span class="sd"> :param expression: An expression filtering the partitions to be returned.</span>
+<span class="sd"> Please see official AWS documentation for further information.</span>
+<span class="sd"> https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-catalog-partitions.html#aws-glue-api-catalog-partitions-GetPartitions</span>
+<span class="sd"> :type expression: str</span>
+<span class="sd"> :param page_size: pagination size</span>
+<span class="sd"> :type page_size: int</span>
+<span class="sd"> :param max_items: maximum items to return</span>
+<span class="sd"> :type max_items: int</span>
+<span class="sd"> :return: set of partition values where each value is a tuple since</span>
+<span class="sd"> a partition may be composed of multiple columns. For example:</span>
+<span class="sd"> {('2018-01-01','1'), ('2018-01-01','2')}</span>
+<span class="sd"> """</span>
+ <span class="n">config</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'PageSize'</span><span class="p">:</span> <span class="n">page_size</span><span class="p">,</span>
+ <span class="s1">'MaxItems'</span><span class="p">:</span> <span class="n">max_items</span><span class="p">,</span>
+ <span class="p">}</span>
+
+ <span class="n">paginator</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">get_paginator</span><span class="p">(</span><span class="s1">'get_partitions'</span><span class="p">)</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="n">paginator</span><span class="o">.</span><span class="n">paginate</span><span class="p">(</span>
+ <span class="n">DatabaseName</span><span class="o">=</span><span class="n">database_name</span><span class="p">,</span>
+ <span class="n">TableName</span><span class="o">=</span><span class="n">table_name</span><span class="p">,</span>
+ <span class="n">Expression</span><span class="o">=</span><span class="n">expression</span><span class="p">,</span>
+ <span class="n">PaginationConfig</span><span class="o">=</span><span class="n">config</span>
+ <span class="p">)</span>
+
+ <span class="n">partitions</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
+ <span class="k">for</span> <span class="n">page</span> <span class="ow">in</span> <span class="n">response</span><span class="p">:</span>
+ <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">page</span><span class="p">[</span><span class="s1">'Partitions'</span><span class="p">]:</span>
+ <span class="n">partitions</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="nb">tuple</span><span class="p">(</span><span class="n">p</span><span class="p">[</span><span class="s1">'Values'</span><span class="p">]))</span>
+
+ <span class="k">return</span> <span class="n">partitions</span></div>
+
+<div class="viewcode-block" id="AwsGlueCatalogHook.check_for_partition"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_glue_catalog_hook.AwsGlueCatalogHook.check_for_partition">[docs]</a> <span class="k">def</span> <span class="nf">check_for_partition</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">database_name</span><span class="p">,</span> <span class="n">table_name</span><span class="p">,</span> <sp [...]
+ <span class="sd">"""</span>
+<span class="sd"> Checks whether a partition exists</span>
+
+<span class="sd"> :param database_name: Name of hive database (schema) @table belongs to</span>
+<span class="sd"> :type database_name: str</span>
+<span class="sd"> :param table_name: Name of hive table @partition belongs to</span>
+<span class="sd"> :type table_name: str</span>
+<span class="sd"> :expression: Expression that matches the partitions to check for</span>
+<span class="sd"> (eg `a = 'b' AND c = 'd'`)</span>
+<span class="sd"> :type expression: str</span>
+<span class="sd"> :rtype: bool</span>
+
+<span class="sd"> >>> hook = AwsGlueCatalogHook()</span>
+<span class="sd"> >>> t = 'static_babynames_partitioned'</span>
+<span class="sd"> >>> hook.check_for_partition('airflow', t, "ds='2015-01-01'")</span>
+<span class="sd"> True</span>
+<span class="sd"> """</span>
+ <span class="n">partitions</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_partitions</span><span class="p">(</span><span class="n">database_name</span><span class="p">,</span> <span class="n">table_name</span><span class="p">,</span> <span class="n">expression</span><span class="p">,</span> <span class="n">max_items</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">partitions</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">True</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">False</span></div></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+ <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/aws_hook.html b/1.10.2/_modules/airflow/contrib/hooks/aws_hook.html
new file mode 100644
index 0000000..eb874a2
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/aws_hook.html
@@ -0,0 +1,424 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.aws_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+ <div class="version">
+ 1.10.2
+ </div>
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.aws_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.aws_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">boto3</span>
+<span class="kn">import</span> <span class="nn">configparser</span>
+<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+
+
+<span class="k">def</span> <span class="nf">_parse_s3_config</span><span class="p">(</span><span class="n">config_file_name</span><span class="p">,</span> <span class="n">config_format</span><span class="o">=</span><span class="s1">'boto'</span><span class="p">,</span> <span class="n">profile</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Parses a config file for s3 credentials. Can currently</span>
+<span class="sd"> parse boto, s3cmd.conf and AWS SDK config formats</span>
+
+<span class="sd"> :param config_file_name: path to the config file</span>
+<span class="sd"> :type config_file_name: str</span>
+<span class="sd"> :param config_format: config type. One of "boto", "s3cmd" or "aws".</span>
+<span class="sd"> Defaults to "boto"</span>
+<span class="sd"> :type config_format: str</span>
+<span class="sd"> :param profile: profile name in AWS type config file</span>
+<span class="sd"> :type profile: str</span>
+<span class="sd"> """</span>
+ <span class="n">config</span> <span class="o">=</span> <span class="n">configparser</span><span class="o">.</span><span class="n">ConfigParser</span><span class="p">()</span>
+ <span class="k">if</span> <span class="n">config</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="n">config_file_name</span><span class="p">):</span> <span class="c1"># pragma: no cover</span>
+ <span class="n">sections</span> <span class="o">=</span> <span class="n">config</span><span class="o">.</span><span class="n">sections</span><span class="p">()</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Couldn't read </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">config_file_name</span><span class="p">))</span>
+ <span class="c1"># Setting option names depending on file format</span>
+ <span class="k">if</span> <span class="n">config_format</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">config_format</span> <span class="o">=</span> <span class="s1">'boto'</span>
+ <span class="n">conf_format</span> <span class="o">=</span> <span class="n">config_format</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
+ <span class="k">if</span> <span class="n">conf_format</span> <span class="o">==</span> <span class="s1">'boto'</span><span class="p">:</span> <span class="c1"># pragma: no cover</span>
+ <span class="k">if</span> <span class="n">profile</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="s1">'profile '</span> <span class="o">+</span> <span class="n">profile</span> <span class="ow">in</span> <span class="n">sections</span><span class="p">:</span>
+ <span class="n">cred_section</span> <span class="o">=</span> <span class="s1">'profile '</span> <span class="o">+</span> <span class="n">profile</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">cred_section</span> <span class="o">=</span> <span class="s1">'Credentials'</span>
+ <span class="k">elif</span> <span class="n">conf_format</span> <span class="o">==</span> <span class="s1">'aws'</span> <span class="ow">and</span> <span class="n">profile</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">cred_section</span> <span class="o">=</span> <span class="n">profile</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">cred_section</span> <span class="o">=</span> <span class="s1">'default'</span>
+ <span class="c1"># Option names</span>
+ <span class="k">if</span> <span class="n">conf_format</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'boto'</span><span class="p">,</span> <span class="s1">'aws'</span><span class="p">):</span> <span class="c1"># pragma: no cover</span>
+ <span class="n">key_id_option</span> <span class="o">=</span> <span class="s1">'aws_access_key_id'</span>
+ <span class="n">secret_key_option</span> <span class="o">=</span> <span class="s1">'aws_secret_access_key'</span>
+ <span class="c1"># security_token_option = 'aws_security_token'</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">key_id_option</span> <span class="o">=</span> <span class="s1">'access_key'</span>
+ <span class="n">secret_key_option</span> <span class="o">=</span> <span class="s1">'secret_key'</span>
+ <span class="c1"># Actual Parsing</span>
+ <span class="k">if</span> <span class="n">cred_section</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">sections</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"This config file format is not recognized"</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">access_key</span> <span class="o">=</span> <span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">cred_section</span><span class="p">,</span> <span class="n">key_id_option</span><span class="p">)</span>
+ <span class="n">secret_key</span> <span class="o">=</span> <span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">cred_section</span><span class="p">,</span> <span class="n">secret_key_option</span><span class="p">)</span>
+ <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
+ <span class="n">logging</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Option Error in parsing s3 config file"</span><span class="p">)</span>
+ <span class="k">raise</span>
+ <span class="k">return</span> <span class="n">access_key</span><span class="p">,</span> <span class="n">secret_key</span>
+
+
+<div class="viewcode-block" id="AwsHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_hook.AwsHook">[docs]</a><span class="k">class</span> <span class="nc">AwsHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with AWS.</span>
+<span class="sd"> This class is a thin wrapper around the boto3 python library.</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">aws_conn_id</span><span class="o">=</span><span class="s1">'aws_default'</span><span class="p">,</span> <span class="n">verify</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span> <span class="o">=</span> <span class="n">aws_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">verify</span> <span class="o">=</span> <span class="n">verify</span>
+
+ <span class="k">def</span> <span class="nf">_get_credentials</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">region_name</span><span class="p">):</span>
+ <span class="n">aws_access_key_id</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="n">aws_secret_access_key</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="n">aws_session_token</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="n">endpoint_url</span> <span class="o">=</span> <span class="kc">None</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">:</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">connection_object</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span>
+ <span class="n">extra_config</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span>
+ <span class="k">if</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">login</span><span class="p">:</span>
+ <span class="n">aws_access_key_id</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">login</span>
+ <span class="n">aws_secret_access_key</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">password</span>
+
+ <span class="k">elif</span> <span class="s1">'aws_secret_access_key'</span> <span class="ow">in</span> <span class="n">extra_config</span><span class="p">:</span>
+ <span class="n">aws_access_key_id</span> <span class="o">=</span> <span class="n">extra_config</span><span class="p">[</span>
+ <span class="s1">'aws_access_key_id'</span><span class="p">]</span>
+ <span class="n">aws_secret_access_key</span> <span class="o">=</span> <span class="n">extra_config</span><span class="p">[</span>
+ <span class="s1">'aws_secret_access_key'</span><span class="p">]</span>
+
+ <span class="k">elif</span> <span class="s1">'s3_config_file'</span> <span class="ow">in</span> <span class="n">extra_config</span><span class="p">:</span>
+ <span class="n">aws_access_key_id</span><span class="p">,</span> <span class="n">aws_secret_access_key</span> <span class="o">=</span> \
+ <span class="n">_parse_s3_config</span><span class="p">(</span>
+ <span class="n">extra_config</span><span class="p">[</span><span class="s1">'s3_config_file'</span><span class="p">],</span>
+ <span class="n">extra_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'s3_config_format'</span><span class="p">),</span>
+ <span class="n">extra_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'profile'</span><span class="p">))</span>
+
+ <span class="k">if</span> <span class="n">region_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">region_name</span> <span class="o">=</span> <span class="n">extra_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'region_name'</span><span class="p">)</span>
+
+ <span class="n">role_arn</span> <span class="o">=</span> <span class="n">extra_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'role_arn'</span><span class="p">)</span>
+ <span class="n">external_id</span> <span class="o">=</span> <span class="n">extra_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'external_id'</span><span class="p">)</span>
+ <span class="n">aws_account_id</span> <span class="o">=</span> <span class="n">extra_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'aws_account_id'</span><span class="p">)</span>
+ <span class="n">aws_iam_role</span> <span class="o">=</span> <span class="n">extra_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'aws_iam_role'</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">role_arn</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">aws_account_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> \
+ <span class="n">aws_iam_role</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">role_arn</span> <span class="o">=</span> <span class="s2">"arn:aws:iam::</span><span class="si">{}</span><span class="s2">:role/</span><span class="si">{}</span><span class="s2">"</span> \
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">aws_account_id</span><span class="p">,</span> <span class="n">aws_iam_role</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">role_arn</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">sts_session</span> <span class="o">=</span> <span class="n">boto3</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">Session</span><span class="p">(</span>
+ <span class="n">aws_access_key_id</span><span class="o">=</span><span class="n">aws_access_key_id</span><span class="p">,</span>
+ <span class="n">aws_secret_access_key</span><span class="o">=</span><span class="n">aws_secret_access_key</span><span class="p">,</span>
+ <span class="n">region_name</span><span class="o">=</span><span class="n">region_name</span><span class="p">)</span>
+
+ <span class="n">sts_client</span> <span class="o">=</span> <span class="n">sts_session</span><span class="o">.</span><span class="n">client</span><span class="p">(</span><span class="s1">'sts'</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">external_id</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">sts_response</span> <span class="o">=</span> <span class="n">sts_client</span><span class="o">.</span><span class="n">assume_role</span><span class="p">(</span>
+ <span class="n">RoleArn</span><span class="o">=</span><span class="n">role_arn</span><span class="p">,</span>
+ <span class="n">RoleSessionName</span><span class="o">=</span><span class="s1">'Airflow_'</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">sts_response</span> <span class="o">=</span> <span class="n">sts_client</span><span class="o">.</span><span class="n">assume_role</span><span class="p">(</span>
+ <span class="n">RoleArn</span><span class="o">=</span><span class="n">role_arn</span><span class="p">,</span>
+ <span class="n">RoleSessionName</span><span class="o">=</span><span class="s1">'Airflow_'</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">,</span>
+ <span class="n">ExternalId</span><span class="o">=</span><span class="n">external_id</span><span class="p">)</span>
+
+ <span class="n">credentials</span> <span class="o">=</span> <span class="n">sts_response</span><span class="p">[</span><span class="s1">'Credentials'</span><span class="p">]</span>
+ <span class="n">aws_access_key_id</span> <span class="o">=</span> <span class="n">credentials</span><span class="p">[</span><span class="s1">'AccessKeyId'</span><span class="p">]</span>
+ <span class="n">aws_secret_access_key</span> <span class="o">=</span> <span class="n">credentials</span><span class="p">[</span><span class="s1">'SecretAccessKey'</span><span class="p">]</span>
+ <span class="n">aws_session_token</span> <span class="o">=</span> <span class="n">credentials</span><span class="p">[</span><span class="s1">'SessionToken'</span><span class="p">]</span>
+
+ <span class="n">endpoint_url</span> <span class="o">=</span> <span class="n">extra_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'host'</span><span class="p">)</span>
+
+ <span class="k">except</span> <span class="n">AirflowException</span><span class="p">:</span>
+ <span class="c1"># No connection found: fallback on boto3 credential strategy</span>
+ <span class="c1"># http://boto3.readthedocs.io/en/latest/guide/configuration.html</span>
+ <span class="k">pass</span>
+
+ <span class="k">return</span> <span class="n">boto3</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">Session</span><span class="p">(</span>
+ <span class="n">aws_access_key_id</span><span class="o">=</span><span class="n">aws_access_key_id</span><span class="p">,</span>
+ <span class="n">aws_secret_access_key</span><span class="o">=</span><span class="n">aws_secret_access_key</span><span class="p">,</span>
+ <span class="n">aws_session_token</span><span class="o">=</span><span class="n">aws_session_token</span><span class="p">,</span>
+ <span class="n">region_name</span><span class="o">=</span><span class="n">region_name</span><span class="p">),</span> <span class="n">endpoint_url</span>
+
+ <span class="k">def</span> <span class="nf">get_client_type</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">client_type</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="n">session</span><span class="p">,</span> <span class="n">endpoint_url</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_credentials</span><span class="p">(</span><span class="n">region_name</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">client</span><span class="p">(</span><span class="n">client_type</span><span class="p">,</span> <span class="n">endpoint_url</span><span class="o">=</span><span class="n">endpoint_url</span><span class="p">,</span>
+ <span class="n">config</span><span class="o">=</span><span class="n">config</span><span class="p">,</span> <span class="n">verify</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verify</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">get_resource_type</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resource_type</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="n">session</span><span class="p">,</span> <span class="n">endpoint_url</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_credentials</span><span class="p">(</span><span class="n">region_name</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">resource</span><span class="p">(</span><span class="n">resource_type</span><span class="p">,</span> <span class="n">endpoint_url</span><span class="o">=</span><span class="n">endpoint_url</span><span class="p">,</span>
+ <span class="n">config</span><span class="o">=</span><span class="n">config</span><span class="p">,</span> <span class="n">verify</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verify</span><span class="p">)</span>
+
+<div class="viewcode-block" id="AwsHook.get_session"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_hook.AwsHook.get_session">[docs]</a> <span class="k">def</span> <span class="nf">get_session</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""Get the underlying boto3.session."""</span>
+ <span class="n">session</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_credentials</span><span class="p">(</span><span class="n">region_name</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">session</span></div>
+
+<div class="viewcode-block" id="AwsHook.get_credentials"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_hook.AwsHook.get_credentials">[docs]</a> <span class="k">def</span> <span class="nf">get_credentials</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""Get the underlying `botocore.Credentials` object.</span>
+
+<span class="sd"> This contains the following authentication attributes: access_key, secret_key and token.</span>
+<span class="sd"> """</span>
+ <span class="n">session</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_credentials</span><span class="p">(</span><span class="n">region_name</span><span class="p">)</span>
+ <span class="c1"># Credentials are refreshable, so accessing your access key and</span>
+ <span class="c1"># secret key separately can lead to a race condition.</span>
+ <span class="c1"># See https://stackoverflow.com/a/36291428/8283373</span>
+ <span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">get_credentials</span><span class="p">()</span><span class="o">.</span><span class="n">get_frozen_credentials</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="AwsHook.expand_role"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_hook.AwsHook.expand_role">[docs]</a> <span class="k">def</span> <span class="nf">expand_role</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">role</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> If the IAM role is a role name, get the Amazon Resource Name (ARN) for the role.</span>
+<span class="sd"> If IAM role is already an IAM role ARN, no change is made.</span>
+
+<span class="sd"> :param role: IAM role name or ARN</span>
+<span class="sd"> :return: IAM role ARN</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="s1">'/'</span> <span class="ow">in</span> <span class="n">role</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">role</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_client_type</span><span class="p">(</span><span class="s1">'iam'</span><span class="p">)</span><span class="o">.</span><span class="n">get_role</span><span class="p">(</span><span class="n">RoleName</span><span class="o">=</span><span class="n">role</span><span class="p">)[</span><span class="s1">'Role'</span><span class="p">][</span><span class="s1">'Arn& [...]
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+ <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/aws_lambda_hook.html b/1.10.2/_modules/airflow/contrib/hooks/aws_lambda_hook.html
new file mode 100644
index 0000000..127ff53
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/aws_lambda_hook.html
@@ -0,0 +1,287 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.aws_lambda_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+ <div class="version">
+ 1.10.2
+ </div>
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.aws_lambda_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.aws_lambda_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.aws_hook</span> <span class="k">import</span> <span class="n">AwsHook</span>
+
+
+<div class="viewcode-block" id="AwsLambdaHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_lambda_hook.AwsLambdaHook">[docs]</a><span class="k">class</span> <span class="nc">AwsLambdaHook</span><span class="p">(</span><span class="n">AwsHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with AWS Lambda</span>
+
+<span class="sd"> :param function_name: AWS Lambda Function Name</span>
+<span class="sd"> :type function_name: str</span>
+<span class="sd"> :param region_name: AWS Region Name (example: us-west-2)</span>
+<span class="sd"> :type region_name: str</span>
+<span class="sd"> :param log_type: Tail Invocation Request</span>
+<span class="sd"> :type log_type: str</span>
+<span class="sd"> :param qualifier: AWS Lambda Function Version or Alias Name</span>
+<span class="sd"> :type qualifier: str</span>
+<span class="sd"> :param invocation_type: AWS Lambda Invocation Type (RequestResponse, Event etc)</span>
+<span class="sd"> :type invocation_type: str</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">function_name</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">log_type</span><span class="o">=</span><span class="s1">'None'</span><span class="p">,</span> <span class="n">qualifier</span><span class="o">=</span><span class="s1">'$LATEST'</span><span class="p">,</span>
+ <span class="n">invocation_type</span><span class="o">=</span><span class="s1">'RequestResponse'</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">function_name</span> <span class="o">=</span> <span class="n">function_name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">region_name</span> <span class="o">=</span> <span class="n">region_name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log_type</span> <span class="o">=</span> <span class="n">log_type</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">invocation_type</span> <span class="o">=</span> <span class="n">invocation_type</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">qualifier</span> <span class="o">=</span> <span class="n">qualifier</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">AwsLambdaHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_client_type</span><span class="p">(</span><span class="s1">'lambda'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">region_name</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span>
+
+<div class="viewcode-block" id="AwsLambdaHook.invoke_lambda"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_lambda_hook.AwsLambdaHook.invoke_lambda">[docs]</a> <span class="k">def</span> <span class="nf">invoke_lambda</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">payload</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Invoke Lambda Function</span>
+<span class="sd"> """</span>
+
+ <span class="n">awslambda_conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+ <span class="n">response</span> <span class="o">=</span> <span class="n">awslambda_conn</span><span class="o">.</span><span class="n">invoke</span><span class="p">(</span>
+ <span class="n">FunctionName</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">function_name</span><span class="p">,</span>
+ <span class="n">InvocationType</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">invocation_type</span><span class="p">,</span>
+ <span class="n">LogType</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">log_type</span><span class="p">,</span>
+ <span class="n">Payload</span><span class="o">=</span><span class="n">payload</span><span class="p">,</span>
+ <span class="n">Qualifier</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">qualifier</span>
+ <span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">response</span></div></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+ <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/azure_cosmos_hook.html b/1.10.2/_modules/airflow/contrib/hooks/azure_cosmos_hook.html
new file mode 100644
index 0000000..72c04e5
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/azure_cosmos_hook.html
@@ -0,0 +1,506 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.azure_cosmos_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+ <div class="version">
+ 1.10.2
+ </div>
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.azure_cosmos_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.azure_cosmos_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">import</span> <span class="nn">azure.cosmos.cosmos_client</span> <span class="k">as</span> <span class="nn">cosmos_client</span>
+<span class="kn">from</span> <span class="nn">azure.cosmos.errors</span> <span class="k">import</span> <span class="n">HTTPFailure</span>
+<span class="kn">import</span> <span class="nn">uuid</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowBadRequest</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+
+
+<div class="viewcode-block" id="AzureCosmosDBHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook">[docs]</a><span class="k">class</span> <span class="nc">AzureCosmosDBHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interacts with Azure CosmosDB.</span>
+
+<span class="sd"> login should be the endpoint uri, password should be the master key</span>
+<span class="sd"> optionally, you can use the following extras to default these values</span>
+<span class="sd"> {"database_name": "<DATABASE_NAME>", "collection_name": "COLLECTION_NAME"}.</span>
+
+<span class="sd"> :param azure_cosmos_conn_id: Reference to the Azure CosmosDB connection.</span>
+<span class="sd"> :type azure_cosmos_conn_id: str</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">azure_cosmos_conn_id</span><span class="o">=</span><span class="s1">'azure_cosmos_default'</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span> <span class="o">=</span> <span class="n">azure_cosmos_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">connection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">extras</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">extra_dejson</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">endpoint_uri</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">login</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">master_key</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">password</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">default_database_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">extras</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'database_name'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">default_collection_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">extras</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'collection_name'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cosmos_client</span> <span class="o">=</span> <span class="kc">None</span>
+
+<div class="viewcode-block" id="AzureCosmosDBHook.get_conn"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Return a cosmos db client.</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cosmos_client</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">cosmos_client</span>
+
+ <span class="c1"># Initialize the Python Azure Cosmos DB client</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cosmos_client</span> <span class="o">=</span> <span class="n">cosmos_client</span><span class="o">.</span><span class="n">CosmosClient</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">endpoint_uri</span><span class="p">,</span> <span class="p">{</span><span class="s1">'masterKey'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</ [...]
+
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">cosmos_client</span></div>
+
+ <span class="k">def</span> <span class="nf">__get_database_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">database_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="n">db_name</span> <span class="o">=</span> <span class="n">database_name</span>
+ <span class="k">if</span> <span class="n">db_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">db_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_database_name</span>
+
+ <span class="k">if</span> <span class="n">db_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">"Database name must be specified"</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">db_name</span>
+
+ <span class="k">def</span> <span class="nf">__get_collection_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">collection_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="n">coll_name</span> <span class="o">=</span> <span class="n">collection_name</span>
+ <span class="k">if</span> <span class="n">coll_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">coll_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_collection_name</span>
+
+ <span class="k">if</span> <span class="n">coll_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">"Collection name must be specified"</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">coll_name</span>
+
+<div class="viewcode-block" id="AzureCosmosDBHook.does_collection_exist"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook.does_collection_exist">[docs]</a> <span class="k">def</span> <span class="nf">does_collection_exist</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">collection_name</span><span class="p">,</span> <span class="n">database_name</span><span class="o"> [...]
+ <span class="sd">"""</span>
+<span class="sd"> Checks if a collection exists in CosmosDB.</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">collection_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">"Collection name cannot be None."</span><span class="p">)</span>
+
+ <span class="n">existing_container</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">QueryContainers</span><span class="p">(</span>
+ <span class="n">get_database_link</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__get_database_name</span><span class="p">(</span><span class="n">database_name</span><span class="p">)),</span> <span class="p">{</span>
+ <span class="s2">"query"</span><span class="p">:</span> <span class="s2">"SELECT * FROM r WHERE r.id=@id"</span><span class="p">,</span>
+ <span class="s2">"parameters"</span><span class="p">:</span> <span class="p">[</span>
+ <span class="p">{</span><span class="s2">"name"</span><span class="p">:</span> <span class="s2">"@id"</span><span class="p">,</span> <span class="s2">"value"</span><span class="p">:</span> <span class="n">collection_name</span><span class="p">}</span>
+ <span class="p">]</span>
+ <span class="p">}))</span>
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing_container</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">False</span>
+
+ <span class="k">return</span> <span class="kc">True</span></div>
+
+<div class="viewcode-block" id="AzureCosmosDBHook.create_collection"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook.create_collection">[docs]</a> <span class="k">def</span> <span class="nf">create_collection</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">collection_name</span><span class="p">,</span> <span class="n">database_name</span><span class="o">=</span><spa [...]
+ <span class="sd">"""</span>
+<span class="sd"> Creates a new collection in the CosmosDB database.</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">collection_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">"Collection name cannot be None."</span><span class="p">)</span>
+
+ <span class="c1"># We need to check to see if this container already exists so we don't try</span>
+ <span class="c1"># to create it twice</span>
+ <span class="n">existing_container</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">QueryContainers</span><span class="p">(</span>
+ <span class="n">get_database_link</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__get_database_name</span><span class="p">(</span><span class="n">database_name</span><span class="p">)),</span> <span class="p">{</span>
+ <span class="s2">"query"</span><span class="p">:</span> <span class="s2">"SELECT * FROM r WHERE r.id=@id"</span><span class="p">,</span>
+ <span class="s2">"parameters"</span><span class="p">:</span> <span class="p">[</span>
+ <span class="p">{</span><span class="s2">"name"</span><span class="p">:</span> <span class="s2">"@id"</span><span class="p">,</span> <span class="s2">"value"</span><span class="p">:</span> <span class="n">collection_name</span><span class="p">}</span>
+ <span class="p">]</span>
+ <span class="p">}))</span>
+
+ <span class="c1"># Only create if we did not find it already existing</span>
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing_container</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">CreateContainer</span><span class="p">(</span>
+ <span class="n">get_database_link</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__get_database_name</span><span class="p">(</span><span class="n">database_name</span><span class="p">)),</span>
+ <span class="p">{</span><span class="s2">"id"</span><span class="p">:</span> <span class="n">collection_name</span><span class="p">})</span></div>
+
+<div class="viewcode-block" id="AzureCosmosDBHook.does_database_exist"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook.does_database_exist">[docs]</a> <span class="k">def</span> <span class="nf">does_database_exist</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">database_name</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Checks if a database exists in CosmosDB.</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">database_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">"Database name cannot be None."</span><span class="p">)</span>
+
+ <span class="n">existing_database</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">QueryDatabases</span><span class="p">({</span>
+ <span class="s2">"query"</span><span class="p">:</span> <span class="s2">"SELECT * FROM r WHERE r.id=@id"</span><span class="p">,</span>
+ <span class="s2">"parameters"</span><span class="p">:</span> <span class="p">[</span>
+ <span class="p">{</span><span class="s2">"name"</span><span class="p">:</span> <span class="s2">"@id"</span><span class="p">,</span> <span class="s2">"value"</span><span class="p">:</span> <span class="n">database_name</span><span class="p">}</span>
+ <span class="p">]</span>
+ <span class="p">}))</span>
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing_database</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">False</span>
+
+ <span class="k">return</span> <span class="kc">True</span></div>
+
+<div class="viewcode-block" id="AzureCosmosDBHook.create_database"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook.create_database">[docs]</a> <span class="k">def</span> <span class="nf">create_database</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">database_name</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Creates a new database in CosmosDB.</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">database_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">"Database name cannot be None."</span><span class="p">)</span>
+
+ <span class="c1"># We need to check to see if this database already exists so we don't try</span>
+ <span class="c1"># to create it twice</span>
+ <span class="n">existing_database</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">QueryDatabases</span><span class="p">({</span>
+ <span class="s2">"query"</span><span class="p">:</span> <span class="s2">"SELECT * FROM r WHERE r.id=@id"</span><span class="p">,</span>
+ <span class="s2">"parameters"</span><span class="p">:</span> <span class="p">[</span>
+ <span class="p">{</span><span class="s2">"name"</span><span class="p">:</span> <span class="s2">"@id"</span><span class="p">,</span> <span class="s2">"value"</span><span class="p">:</span> <span class="n">database_name</span><span class="p">}</span>
+ <span class="p">]</span>
+ <span class="p">}))</span>
+
+ <span class="c1"># Only create if we did not find it already existing</span>
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing_database</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">CreateDatabase</span><span class="p">({</span><span class="s2">"id"</span><span class="p">:</span> <span class="n">database_name</span><span class="p">})</span></div>
+
+<div class="viewcode-block" id="AzureCosmosDBHook.delete_database"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook.delete_database">[docs]</a> <span class="k">def</span> <span class="nf">delete_database</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">database_name</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Deletes an existing database in CosmosDB.</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">database_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">"Database name cannot be None."</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">DeleteDatabase</span><span class="p">(</span><span class="n">get_database_link</span><span class="p">(</span><span class="n">database_name</span><span class="p">))</span></div>
+
+<div class="viewcode-block" id="AzureCosmosDBHook.delete_collection"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook.delete_collection">[docs]</a> <span class="k">def</span> <span class="nf">delete_collection</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">collection_name</span><span class="p">,</span> <span class="n">database_name</span><span class="o">=</span><spa [...]
+ <span class="sd">"""</span>
+<span class="sd"> Deletes an existing collection in the CosmosDB database.</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">collection_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">"Collection name cannot be None."</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">DeleteContainer</span><span class="p">(</span>
+ <span class="n">get_collection_link</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__get_database_name</span><span class="p">(</span><span class="n">database_name</span><span class="p">),</span> <span class="n">collection_name</span><span class="p">))</span></div>
+
+<div class="viewcode-block" id="AzureCosmosDBHook.upsert_document"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook.upsert_document">[docs]</a> <span class="k">def</span> <span class="nf">upsert_document</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">document</span><span class="p">,</span> <span class="n">database_name</span><span class="o">=</span><span class="kc"> [...]
+ <span class="sd">"""</span>
+<span class="sd"> Inserts a new document (or updates an existing one) into an existing</span>
+<span class="sd"> collection in the CosmosDB database.</span>
+<span class="sd"> """</span>
+ <span class="c1"># Assign unique ID if one isn't provided</span>
+ <span class="k">if</span> <span class="n">document_id</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">document_id</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">uuid</span><span class="o">.</span><span class="n">uuid4</span><span class="p">())</span>
+
+ <span class="k">if</span> <span class="n">document</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">"You cannot insert a None document"</span><span class="p">)</span>
+
+ <span class="c1"># Add document id if isn't found</span>
+ <span class="k">if</span> <span class="s1">'id'</span> <span class="ow">in</span> <span class="n">document</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">document</span><span class="p">[</span><span class="s1">'id'</span><span class="p">]</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">document</span><span class="p">[</span><span class="s1">'id'</span><span class="p">]</span> <span class="o">=</span> <span class="n">document_id</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">document</span><span class="p">[</span><span class="s1">'id'</span><span class="p">]</span> <span class="o">=</span> <span class="n">document_id</span>
+
+ <span class="n">created_document</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">CreateItem</span><span class="p">(</span>
+ <span class="n">get_collection_link</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">__get_database_name</span><span class="p">(</span><span class="n">database_name</span><span class="p">),</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">__get_collection_name</span><span class="p">(</span><span class="n">collection_name</span><span class="p">)),</span>
+ <span class="n">document</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">created_document</span></div>
+
+<div class="viewcode-block" id="AzureCosmosDBHook.insert_documents"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook.insert_documents">[docs]</a> <span class="k">def</span> <span class="nf">insert_documents</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">documents</span><span class="p">,</span> <span class="n">database_name</span><span class="o">=</span><span class=" [...]
+ <span class="sd">"""</span>
+<span class="sd"> Insert a list of new documents into an existing collection in the CosmosDB database.</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">documents</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">"You cannot insert empty documents"</span><span class="p">)</span>
+
+ <span class="n">created_documents</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="k">for</span> <span class="n">single_document</span> <span class="ow">in</span> <span class="n">documents</span><span class="p">:</span>
+ <span class="n">created_documents</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">CreateItem</span><span class="p">(</span>
+ <span class="n">get_collection_link</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">__get_database_name</span><span class="p">(</span><span class="n">database_name</span><span class="p">),</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">__get_collection_name</span><span class="p">(</span><span class="n">collection_name</span><span class="p">)),</span>
+ <span class="n">single_document</span><span class="p">))</span>
+
+ <span class="k">return</span> <span class="n">created_documents</span></div>
+
+<div class="viewcode-block" id="AzureCosmosDBHook.delete_document"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook.delete_document">[docs]</a> <span class="k">def</span> <span class="nf">delete_document</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">document_id</span><span class="p">,</span> <span class="n">database_name</span><span class="o">=</span><span class="k [...]
+ <span class="sd">"""</span>
+<span class="sd"> Delete an existing document out of a collection in the CosmosDB database.</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">document_id</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">"Cannot delete a document without an id"</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">DeleteItem</span><span class="p">(</span>
+ <span class="n">get_document_link</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">__get_database_name</span><span class="p">(</span><span class="n">database_name</span><span class="p">),</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">__get_collection_name</span><span class="p">(</span><span class="n">collection_name</span><span class="p">),</span>
+ <span class="n">document_id</span><span class="p">))</span></div>
+
+<div class="viewcode-block" id="AzureCosmosDBHook.get_document"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook.get_document">[docs]</a> <span class="k">def</span> <span class="nf">get_document</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">document_id</span><span class="p">,</span> <span class="n">database_name</span><span class="o">=</span><span class="kc">None</ [...]
+ <span class="sd">"""</span>
+<span class="sd"> Get a document from an existing collection in the CosmosDB database.</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">document_id</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">"Cannot get a document without an id"</span><span class="p">)</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">ReadItem</span><span class="p">(</span>
+ <span class="n">get_document_link</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">__get_database_name</span><span class="p">(</span><span class="n">database_name</span><span class="p">),</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">__get_collection_name</span><span class="p">(</span><span class="n">collection_name</span><span class="p">),</span>
+ <span class="n">document_id</span><span class="p">))</span>
+ <span class="k">except</span> <span class="n">HTTPFailure</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">None</span></div>
+
+<div class="viewcode-block" id="AzureCosmosDBHook.get_documents"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook.get_documents">[docs]</a> <span class="k">def</span> <span class="nf">get_documents</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql_string</span><span class="p">,</span> <span class="n">database_name</span><span class="o">=</span><span class="kc">None [...]
+ <span class="sd">"""</span>
+<span class="sd"> Get a list of documents from an existing collection in the CosmosDB database via SQL query.</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">sql_string</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">"SQL query string cannot be None"</span><span class="p">)</span>
+
+ <span class="c1"># Query them in SQL</span>
+ <span class="n">query</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'query'</span><span class="p">:</span> <span class="n">sql_string</span><span class="p">}</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">result_iterable</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">QueryItems</span><span class="p">(</span>
+ <span class="n">get_collection_link</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">__get_database_name</span><span class="p">(</span><span class="n">database_name</span><span class="p">),</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">__get_collection_name</span><span class="p">(</span><span class="n">collection_name</span><span class="p">)),</span>
+ <span class="n">query</span><span class="p">,</span>
+ <span class="n">partition_key</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="n">result_iterable</span><span class="p">)</span>
+ <span class="k">except</span> <span class="n">HTTPFailure</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">None</span></div></div>
+
+
+<span class="k">def</span> <span class="nf">get_database_link</span><span class="p">(</span><span class="n">database_id</span><span class="p">):</span>
+ <span class="k">return</span> <span class="s2">"dbs/"</span> <span class="o">+</span> <span class="n">database_id</span>
+
+
+<span class="k">def</span> <span class="nf">get_collection_link</span><span class="p">(</span><span class="n">database_id</span><span class="p">,</span> <span class="n">collection_id</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">get_database_link</span><span class="p">(</span><span class="n">database_id</span><span class="p">)</span> <span class="o">+</span> <span class="s2">"/colls/"</span> <span class="o">+</span> <span class="n">collection_id</span>
+
+
+<span class="k">def</span> <span class="nf">get_document_link</span><span class="p">(</span><span class="n">database_id</span><span class="p">,</span> <span class="n">collection_id</span><span class="p">,</span> <span class="n">document_id</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">get_collection_link</span><span class="p">(</span><span class="n">database_id</span><span class="p">,</span> <span class="n">collection_id</span><span class="p">)</span> <span class="o">+</span> <span class="s2">"/docs/"</span> <span class="o">+</span> <span class="n">document_id</span>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+ <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/azure_data_lake_hook.html b/1.10.2/_modules/airflow/contrib/hooks/azure_data_lake_hook.html
new file mode 100644
index 0000000..8308899
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/azure_data_lake_hook.html
@@ -0,0 +1,360 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.azure_data_lake_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+ <div class="version">
+ 1.10.2
+ </div>
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.azure_data_lake_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.azure_data_lake_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">azure.datalake.store</span> <span class="k">import</span> <span class="n">core</span><span class="p">,</span> <span class="n">lib</span><span class="p">,</span> <span class="n">multithread</span>
+
+
+<div class="viewcode-block" id="AzureDataLakeHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_data_lake_hook.AzureDataLakeHook">[docs]</a><span class="k">class</span> <span class="nc">AzureDataLakeHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interacts with Azure Data Lake.</span>
+
+<span class="sd"> Client ID and client secret should be in user and password parameters.</span>
+<span class="sd"> Tenant and account name should be extra field as</span>
+<span class="sd"> {"tenant": "<TENANT>", "account_name": "ACCOUNT_NAME"}.</span>
+
+<span class="sd"> :param azure_data_lake_conn_id: Reference to the Azure Data Lake connection.</span>
+<span class="sd"> :type azure_data_lake_conn_id: str</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">azure_data_lake_conn_id</span><span class="o">=</span><span class="s1">'azure_data_lake_default'</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span> <span class="o">=</span> <span class="n">azure_data_lake_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">connection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+<div class="viewcode-block" id="AzureDataLakeHook.get_conn"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_data_lake_hook.AzureDataLakeHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""Return a AzureDLFileSystem object."""</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+ <span class="n">service_options</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">account_name</span> <span class="o">=</span> <span class="n">service_options</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'account_name'</span><span class="p">)</span>
+
+ <span class="n">adlCreds</span> <span class="o">=</span> <span class="n">lib</span><span class="o">.</span><span class="n">auth</span><span class="p">(</span><span class="n">tenant_id</span><span class="o">=</span><span class="n">service_options</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'tenant'</span><span class="p">),</span>
+ <span class="n">client_secret</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span>
+ <span class="n">client_id</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">)</span>
+ <span class="n">adlsFileSystemClient</span> <span class="o">=</span> <span class="n">core</span><span class="o">.</span><span class="n">AzureDLFileSystem</span><span class="p">(</span><span class="n">adlCreds</span><span class="p">,</span>
+ <span class="n">store_name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">account_name</span><span class="p">)</span>
+ <span class="n">adlsFileSystemClient</span><span class="o">.</span><span class="n">connect</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">adlsFileSystemClient</span></div>
+
+<div class="viewcode-block" id="AzureDataLakeHook.check_for_file"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_data_lake_hook.AzureDataLakeHook.check_for_file">[docs]</a> <span class="k">def</span> <span class="nf">check_for_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">file_path</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Check if a file exists on Azure Data Lake.</span>
+
+<span class="sd"> :param file_path: Path and name of the file.</span>
+<span class="sd"> :type file_path: str</span>
+<span class="sd"> :return: True if the file exists, False otherwise.</span>
+<span class="sd"> :rtype: bool</span>
+<span class="sd"> """</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">files</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">glob</span><span class="p">(</span><span class="n">file_path</span><span class="p">,</span> <span class="n">details</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">invalidate_cache</span><span class="o">=</span><span class="kc">True</span><span clas [...]
+ <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="n">files</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span>
+ <span class="k">except</span> <span class="ne">FileNotFoundError</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">False</span></div>
+
+<div class="viewcode-block" id="AzureDataLakeHook.upload_file"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_data_lake_hook.AzureDataLakeHook.upload_file">[docs]</a> <span class="k">def</span> <span class="nf">upload_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">local_path</span><span class="p">,</span> <span class="n">remote_path</span><span class="p">,</span> <span class="n">nthreads< [...]
+ <span class="n">buffersize</span><span class="o">=</span><span class="mi">4194304</span><span class="p">,</span> <span class="n">blocksize</span><span class="o">=</span><span class="mi">4194304</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Upload a file to Azure Data Lake.</span>
+
+<span class="sd"> :param local_path: local path. Can be single file, directory (in which case,</span>
+<span class="sd"> upload recursively) or glob pattern. Recursive glob patterns using `**`</span>
+<span class="sd"> are not supported.</span>
+<span class="sd"> :type local_path: str</span>
+<span class="sd"> :param remote_path: Remote path to upload to; if multiple files, this is the</span>
+<span class="sd"> dircetory root to write within.</span>
+<span class="sd"> :type remote_path: str</span>
+<span class="sd"> :param nthreads: Number of threads to use. If None, uses the number of cores.</span>
+<span class="sd"> :type nthreads: int</span>
+<span class="sd"> :param overwrite: Whether to forcibly overwrite existing files/directories.</span>
+<span class="sd"> If False and remote path is a directory, will quit regardless if any files</span>
+<span class="sd"> would be overwritten or not. If True, only matching filenames are actually</span>
+<span class="sd"> overwritten.</span>
+<span class="sd"> :type overwrite: bool</span>
+<span class="sd"> :param buffersize: int [2**22]</span>
+<span class="sd"> Number of bytes for internal buffer. This block cannot be bigger than</span>
+<span class="sd"> a chunk and cannot be smaller than a block.</span>
+<span class="sd"> :type buffersize: int</span>
+<span class="sd"> :param blocksize: int [2**22]</span>
+<span class="sd"> Number of bytes for a block. Within each chunk, we write a smaller</span>
+<span class="sd"> block for each API call. This block cannot be bigger than a chunk.</span>
+<span class="sd"> :type blocksize: int</span>
+<span class="sd"> """</span>
+ <span class="n">multithread</span><span class="o">.</span><span class="n">ADLUploader</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="p">,</span>
+ <span class="n">lpath</span><span class="o">=</span><span class="n">local_path</span><span class="p">,</span>
+ <span class="n">rpath</span><span class="o">=</span><span class="n">remote_path</span><span class="p">,</span>
+ <span class="n">nthreads</span><span class="o">=</span><span class="n">nthreads</span><span class="p">,</span>
+ <span class="n">overwrite</span><span class="o">=</span><span class="n">overwrite</span><span class="p">,</span>
+ <span class="n">buffersize</span><span class="o">=</span><span class="n">buffersize</span><span class="p">,</span>
+ <span class="n">blocksize</span><span class="o">=</span><span class="n">blocksize</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureDataLakeHook.download_file"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_data_lake_hook.AzureDataLakeHook.download_file">[docs]</a> <span class="k">def</span> <span class="nf">download_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">local_path</span><span class="p">,</span> <span class="n">remote_path</span><span class="p">,</span> <span class="n">nth [...]
+ <span class="n">buffersize</span><span class="o">=</span><span class="mi">4194304</span><span class="p">,</span> <span class="n">blocksize</span><span class="o">=</span><span class="mi">4194304</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Download a file from Azure Blob Storage.</span>
+
+<span class="sd"> :param local_path: local path. If downloading a single file, will write to this</span>
+<span class="sd"> specific file, unless it is an existing directory, in which case a file is</span>
+<span class="sd"> created within it. If downloading multiple files, this is the root</span>
+<span class="sd"> directory to write within. Will create directories as required.</span>
+<span class="sd"> :type local_path: str</span>
+<span class="sd"> :param remote_path: remote path/globstring to use to find remote files.</span>
+<span class="sd"> Recursive glob patterns using `**` are not supported.</span>
+<span class="sd"> :type remote_path: str</span>
+<span class="sd"> :param nthreads: Number of threads to use. If None, uses the number of cores.</span>
+<span class="sd"> :type nthreads: int</span>
+<span class="sd"> :param overwrite: Whether to forcibly overwrite existing files/directories.</span>
+<span class="sd"> If False and remote path is a directory, will quit regardless if any files</span>
+<span class="sd"> would be overwritten or not. If True, only matching filenames are actually</span>
+<span class="sd"> overwritten.</span>
+<span class="sd"> :type overwrite: bool</span>
+<span class="sd"> :param buffersize: int [2**22]</span>
+<span class="sd"> Number of bytes for internal buffer. This block cannot be bigger than</span>
+<span class="sd"> a chunk and cannot be smaller than a block.</span>
+<span class="sd"> :type buffersize: int</span>
+<span class="sd"> :param blocksize: int [2**22]</span>
+<span class="sd"> Number of bytes for a block. Within each chunk, we write a smaller</span>
+<span class="sd"> block for each API call. This block cannot be bigger than a chunk.</span>
+<span class="sd"> :type blocksize: int</span>
+<span class="sd"> """</span>
+ <span class="n">multithread</span><span class="o">.</span><span class="n">ADLDownloader</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="p">,</span>
+ <span class="n">lpath</span><span class="o">=</span><span class="n">local_path</span><span class="p">,</span>
+ <span class="n">rpath</span><span class="o">=</span><span class="n">remote_path</span><span class="p">,</span>
+ <span class="n">nthreads</span><span class="o">=</span><span class="n">nthreads</span><span class="p">,</span>
+ <span class="n">overwrite</span><span class="o">=</span><span class="n">overwrite</span><span class="p">,</span>
+ <span class="n">buffersize</span><span class="o">=</span><span class="n">buffersize</span><span class="p">,</span>
+ <span class="n">blocksize</span><span class="o">=</span><span class="n">blocksize</span><span class="p">)</span></div></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+ <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/azure_fileshare_hook.html b/1.10.2/_modules/airflow/contrib/hooks/azure_fileshare_hook.html
new file mode 100644
index 0000000..a7696ab
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/azure_fileshare_hook.html
@@ -0,0 +1,431 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.azure_fileshare_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+ <div class="version">
+ 1.10.2
+ </div>
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.azure_fileshare_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.azure_fileshare_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">azure.storage.file</span> <span class="k">import</span> <span class="n">FileService</span>
+
+
+<div class="viewcode-block" id="AzureFileShareHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook">[docs]</a><span class="k">class</span> <span class="nc">AzureFileShareHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interacts with Azure FileShare Storage.</span>
+
+<span class="sd"> Additional options passed in the 'extra' field of the connection will be</span>
+<span class="sd"> passed to the `FileService()` constructor.</span>
+
+<span class="sd"> :param wasb_conn_id: Reference to the wasb connection.</span>
+<span class="sd"> :type wasb_conn_id: str</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">wasb_conn_id</span><span class="o">=</span><span class="s1">'wasb_default'</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span> <span class="o">=</span> <span class="n">wasb_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">connection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+<div class="viewcode-block" id="AzureFileShareHook.get_conn"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""Return the FileService object."""</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+ <span class="n">service_options</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span>
+ <span class="k">return</span> <span class="n">FileService</span><span class="p">(</span><span class="n">account_name</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span>
+ <span class="n">account_key</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span> <span class="o">**</span><span class="n">service_options</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.check_for_directory"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.check_for_directory">[docs]</a> <span class="k">def</span> <span class="nf">check_for_directory</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</sp [...]
+ <span class="sd">"""</span>
+<span class="sd"> Check if a directory exists on Azure File Share.</span>
+
+<span class="sd"> :param share_name: Name of the share.</span>
+<span class="sd"> :type share_name: str</span>
+<span class="sd"> :param directory_name: Name of the directory.</span>
+<span class="sd"> :type directory_name: str</span>
+<span class="sd"> :param kwargs: Optional keyword arguments that</span>
+<span class="sd"> `FileService.exists()` takes.</span>
+<span class="sd"> :type kwargs: object</span>
+<span class="sd"> :return: True if the file exists, False otherwise.</span>
+<span class="sd"> :rtype: bool</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.check_for_file"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.check_for_file">[docs]</a> <span class="k">def</span> <span class="nf">check_for_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span> <span class [...]
+ <span class="sd">"""</span>
+<span class="sd"> Check if a file exists on Azure File Share.</span>
+
+<span class="sd"> :param share_name: Name of the share.</span>
+<span class="sd"> :type share_name: str</span>
+<span class="sd"> :param directory_name: Name of the directory.</span>
+<span class="sd"> :type directory_name: str</span>
+<span class="sd"> :param file_name: Name of the file.</span>
+<span class="sd"> :type file_name: str</span>
+<span class="sd"> :param kwargs: Optional keyword arguments that</span>
+<span class="sd"> `FileService.exists()` takes.</span>
+<span class="sd"> :type kwargs: object</span>
+<span class="sd"> :return: True if the file exists, False otherwise.</span>
+<span class="sd"> :rtype: bool</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span>
+ <span class="n">file_name</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.list_directories_and_files"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.list_directories_and_files">[docs]</a> <span class="k">def</span> <span class="nf">list_directories_and_files</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span> [...]
+ <span class="sd">"""</span>
+<span class="sd"> Return the list of directories and files stored on a Azure File Share.</span>
+
+<span class="sd"> :param share_name: Name of the share.</span>
+<span class="sd"> :type share_name: str</span>
+<span class="sd"> :param directory_name: Name of the directory.</span>
+<span class="sd"> :type directory_name: str</span>
+<span class="sd"> :param kwargs: Optional keyword arguments that</span>
+<span class="sd"> `FileService.list_directories_and_files()` takes.</span>
+<span class="sd"> :type kwargs: object</span>
+<span class="sd"> :return: A list of files and directories</span>
+<span class="sd"> :rtype: list</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">list_directories_and_files</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span>
+ <span class="n">directory_name</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.create_directory"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.create_directory">[docs]</a> <span class="k">def</span> <span class="nf">create_directory</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span> <span [...]
+ <span class="sd">"""</span>
+<span class="sd"> Create a new directory on a Azure File Share.</span>
+
+<span class="sd"> :param share_name: Name of the share.</span>
+<span class="sd"> :type share_name: str</span>
+<span class="sd"> :param directory_name: Name of the directory.</span>
+<span class="sd"> :type directory_name: str</span>
+<span class="sd"> :param kwargs: Optional keyword arguments that</span>
+<span class="sd"> `FileService.create_directory()` takes.</span>
+<span class="sd"> :type kwargs: object</span>
+<span class="sd"> :return: A list of files and directories</span>
+<span class="sd"> :rtype: list</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">create_directory</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.get_file"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.get_file">[docs]</a> <span class="k">def</span> <span class="nf">get_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">file_path</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</sp [...]
+ <span class="sd">"""</span>
+<span class="sd"> Download a file from Azure File Share.</span>
+
+<span class="sd"> :param file_path: Where to store the file.</span>
+<span class="sd"> :type file_path: str</span>
+<span class="sd"> :param share_name: Name of the share.</span>
+<span class="sd"> :type share_name: str</span>
+<span class="sd"> :param directory_name: Name of the directory.</span>
+<span class="sd"> :type directory_name: str</span>
+<span class="sd"> :param file_name: Name of the file.</span>
+<span class="sd"> :type file_name: str</span>
+<span class="sd"> :param kwargs: Optional keyword arguments that</span>
+<span class="sd"> `FileService.get_file_to_path()` takes.</span>
+<span class="sd"> :type kwargs: object</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">get_file_to_path</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span>
+ <span class="n">file_name</span><span class="p">,</span> <span class="n">file_path</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.get_file_to_stream"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.get_file_to_stream">[docs]</a> <span class="k">def</span> <span class="nf">get_file_to_stream</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">stream</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span c [...]
+ <span class="sd">"""</span>
+<span class="sd"> Download a file from Azure File Share.</span>
+
+<span class="sd"> :param stream: A filehandle to store the file to.</span>
+<span class="sd"> :type stream: file-like object</span>
+<span class="sd"> :param share_name: Name of the share.</span>
+<span class="sd"> :type share_name: str</span>
+<span class="sd"> :param directory_name: Name of the directory.</span>
+<span class="sd"> :type directory_name: str</span>
+<span class="sd"> :param file_name: Name of the file.</span>
+<span class="sd"> :type file_name: str</span>
+<span class="sd"> :param kwargs: Optional keyword arguments that</span>
+<span class="sd"> `FileService.get_file_to_stream()` takes.</span>
+<span class="sd"> :type kwargs: object</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">get_file_to_stream</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span>
+ <span class="n">file_name</span><span class="p">,</span> <span class="n">stream</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.load_file"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.load_file">[docs]</a> <span class="k">def</span> <span class="nf">load_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">file_path</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name< [...]
+ <span class="sd">"""</span>
+<span class="sd"> Upload a file to Azure File Share.</span>
+
+<span class="sd"> :param file_path: Path to the file to load.</span>
+<span class="sd"> :type file_path: str</span>
+<span class="sd"> :param share_name: Name of the share.</span>
+<span class="sd"> :type share_name: str</span>
+<span class="sd"> :param directory_name: Name of the directory.</span>
+<span class="sd"> :type directory_name: str</span>
+<span class="sd"> :param file_name: Name of the file.</span>
+<span class="sd"> :type file_name: str</span>
+<span class="sd"> :param kwargs: Optional keyword arguments that</span>
+<span class="sd"> `FileService.create_file_from_path()` takes.</span>
+<span class="sd"> :type kwargs: object</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">create_file_from_path</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span>
+ <span class="n">file_name</span><span class="p">,</span> <span class="n">file_path</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.load_string"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.load_string">[docs]</a> <span class="k">def</span> <span class="nf">load_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">string_data</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span class="n">directo [...]
+ <span class="sd">"""</span>
+<span class="sd"> Upload a string to Azure File Share.</span>
+
+<span class="sd"> :param string_data: String to load.</span>
+<span class="sd"> :type string_data: str</span>
+<span class="sd"> :param share_name: Name of the share.</span>
+<span class="sd"> :type share_name: str</span>
+<span class="sd"> :param directory_name: Name of the directory.</span>
+<span class="sd"> :type directory_name: str</span>
+<span class="sd"> :param file_name: Name of the file.</span>
+<span class="sd"> :type file_name: str</span>
+<span class="sd"> :param kwargs: Optional keyword arguments that</span>
+<span class="sd"> `FileService.create_file_from_text()` takes.</span>
+<span class="sd"> :type kwargs: object</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">create_file_from_text</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span>
+ <span class="n">file_name</span><span class="p">,</span> <span class="n">string_data</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.load_stream"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.load_stream">[docs]</a> <span class="k">def</span> <span class="nf">load_stream</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">stream</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span class="n">directory_na [...]
+ <span class="sd">"""</span>
+<span class="sd"> Upload a stream to Azure File Share.</span>
+
+<span class="sd"> :param stream: Opened file/stream to upload as the file content.</span>
+<span class="sd"> :type stream: file-like</span>
+<span class="sd"> :param share_name: Name of the share.</span>
+<span class="sd"> :type share_name: str</span>
+<span class="sd"> :param directory_name: Name of the directory.</span>
+<span class="sd"> :type directory_name: str</span>
+<span class="sd"> :param file_name: Name of the file.</span>
+<span class="sd"> :type file_name: str</span>
+<span class="sd"> :param count: Size of the stream in bytes</span>
+<span class="sd"> :type count: int</span>
+<span class="sd"> :param kwargs: Optional keyword arguments that</span>
+<span class="sd"> `FileService.create_file_from_stream()` takes.</span>
+<span class="sd"> :type kwargs: object</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">create_file_from_stream</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span>
+ <span class="n">file_name</span><span class="p">,</span> <span class="n">stream</span><span class="p">,</span> <span class="n">count</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+ <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/bigquery_hook.html b/1.10.2/_modules/airflow/contrib/hooks/bigquery_hook.html
new file mode 100644
index 0000000..162344f
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/bigquery_hook.html
@@ -0,0 +1,2184 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.bigquery_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+ <div class="version">
+ 1.10.2
+ </div>
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.bigquery_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.bigquery_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+<span class="sd">"""</span>
+<span class="sd">This module contains a BigQuery Hook, as well as a very basic PEP 249</span>
+<span class="sd">implementation for BigQuery.</span>
+<span class="sd">"""</span>
+
+<span class="kn">import</span> <span class="nn">time</span>
+<span class="kn">import</span> <span class="nn">six</span>
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">range</span>
+<span class="kn">from</span> <span class="nn">copy</span> <span class="k">import</span> <span class="n">deepcopy</span>
+<span class="kn">from</span> <span class="nn">six</span> <span class="k">import</span> <span class="n">iteritems</span>
+
+<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="k">import</span> <span class="n">basestring</span>
+
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="k">import</span> <span class="n">GoogleCloudBaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="k">import</span> <span class="n">DbApiHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+<span class="kn">from</span> <span class="nn">googleapiclient.discovery</span> <span class="k">import</span> <span class="n">build</span>
+<span class="kn">from</span> <span class="nn">googleapiclient.errors</span> <span class="k">import</span> <span class="n">HttpError</span>
+<span class="kn">from</span> <span class="nn">pandas_gbq.gbq</span> <span class="k">import</span> \
+ <span class="n">_check_google_client_version</span> <span class="k">as</span> <span class="n">gbq_check_google_client_version</span>
+<span class="kn">from</span> <span class="nn">pandas_gbq</span> <span class="k">import</span> <span class="n">read_gbq</span>
+<span class="kn">from</span> <span class="nn">pandas_gbq.gbq</span> <span class="k">import</span> \
+ <span class="n">_test_google_api_imports</span> <span class="k">as</span> <span class="n">gbq_test_google_api_imports</span>
+<span class="kn">from</span> <span class="nn">pandas_gbq.gbq</span> <span class="k">import</span> <span class="n">GbqConnector</span>
+
+
+<div class="viewcode-block" id="BigQueryHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook">[docs]</a><span class="k">class</span> <span class="nc">BigQueryHook</span><span class="p">(</span><span class="n">GoogleCloudBaseHook</span><span class="p">,</span> <span class="n">DbApiHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with BigQuery. This hook uses the Google Cloud Platform</span>
+<span class="sd"> connection.</span>
+<span class="sd"> """</span>
+ <span class="n">conn_name_attr</span> <span class="o">=</span> <span class="s1">'bigquery_conn_id'</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">'bigquery_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">use_legacy_sql</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+ <span class="n">location</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="n">bigquery_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="o">=</span><span class="n">delegate_to</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span> <span class="o">=</span> <span class="n">use_legacy_sql</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">location</span> <span class="o">=</span> <span class="n">location</span>
+
+<div class="viewcode-block" id="BigQueryHook.get_conn"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns a BigQuery PEP 249 connection object.</span>
+<span class="sd"> """</span>
+ <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_service</span><span class="p">()</span>
+ <span class="n">project</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_field</span><span class="p">(</span><span class="s1">'project'</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">BigQueryConnection</span><span class="p">(</span>
+ <span class="n">service</span><span class="o">=</span><span class="n">service</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="o">=</span><span class="n">project</span><span class="p">,</span>
+ <span class="n">use_legacy_sql</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span><span class="p">,</span>
+ <span class="n">location</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">,</span>
+ <span class="p">)</span></div>
+
+<div class="viewcode-block" id="BigQueryHook.get_service"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook.get_service">[docs]</a> <span class="k">def</span> <span class="nf">get_service</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns a BigQuery service object.</span>
+<span class="sd"> """</span>
+ <span class="n">http_authorized</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">build</span><span class="p">(</span>
+ <span class="s1">'bigquery'</span><span class="p">,</span> <span class="s1">'v2'</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">,</span> <span class="n">cache_discovery</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="BigQueryHook.insert_rows"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook.insert_rows">[docs]</a> <span class="k">def</span> <span class="nf">insert_rows</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">rows</span><span class="p">,</span> <span class="n">target_fields</span><span class="o">=< [...]
+ <span class="sd">"""</span>
+<span class="sd"> Insertion is currently unsupported. Theoretically, you could use</span>
+<span class="sd"> BigQuery's streaming API to insert rows into a table, but this hasn't</span>
+<span class="sd"> been implemented.</span>
+<span class="sd"> """</span>
+ <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="BigQueryHook.get_pandas_df"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook.get_pandas_df">[docs]</a> <span class="k">def</span> <span class="nf">get_pandas_df</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">, [...]
+ <span class="sd">"""</span>
+<span class="sd"> Returns a Pandas DataFrame for the results produced by a BigQuery</span>
+<span class="sd"> query. The DbApiHook method must be overridden because Pandas</span>
+<span class="sd"> doesn't support PEP 249 connections, except for SQLite. See:</span>
+
+<span class="sd"> https://github.com/pydata/pandas/blob/master/pandas/io/sql.py#L447</span>
+<span class="sd"> https://github.com/pydata/pandas/issues/6900</span>
+
+<span class="sd"> :param sql: The BigQuery SQL to execute.</span>
+<span class="sd"> :type sql: string</span>
+<span class="sd"> :param parameters: The parameters to render the SQL query with (not</span>
+<span class="sd"> used, leave to override superclass method)</span>
+<span class="sd"> :type parameters: mapping or iterable</span>
+<span class="sd"> :param dialect: Dialect of BigQuery SQL – legacy SQL or standard SQL</span>
+<span class="sd"> defaults to use `self.use_legacy_sql` if not specified</span>
+<span class="sd"> :type dialect: string in {'legacy', 'standard'}</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">dialect</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">dialect</span> <span class="o">=</span> <span class="s1">'legacy'</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span> <span class="k">else</span> <span class="s1">'standard'</span>
+
+ <span class="k">return</span> <span class="n">read_gbq</span><span class="p">(</span><span class="n">sql</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_get_field</span><span class="p">(</span><span class="s1">'project'</span><span class="p">),</span>
+ <span class="n">dialect</span><span class="o">=</span><span class="n">dialect</span><span class="p">,</span>
+ <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="BigQueryHook.table_exists"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook.table_exists">[docs]</a> <span class="k">def</span> <span class="nf">table_exists</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span cla [...]
+ <span class="sd">"""</span>
+<span class="sd"> Checks for the existence of a table in Google BigQuery.</span>
+
+<span class="sd"> :param project_id: The Google cloud project in which to look for the</span>
+<span class="sd"> table. The connection supplied to the hook must provide access to</span>
+<span class="sd"> the specified project.</span>
+<span class="sd"> :type project_id: string</span>
+<span class="sd"> :param dataset_id: The name of the dataset in which to look for the</span>
+<span class="sd"> table.</span>
+<span class="sd"> :type dataset_id: string</span>
+<span class="sd"> :param table_id: The name of the table to check the existence of.</span>
+<span class="sd"> :type table_id: string</span>
+<span class="sd"> """</span>
+ <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_service</span><span class="p">()</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span> <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+ <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">return</span> <span class="kc">True</span>
+ <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">e</span><span class="o">.</span><span class="n">resp</span><span class="p">[</span><span class="s1">'status'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'404'</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">False</span>
+ <span class="k">raise</span></div></div>
+
+
+<span class="k">class</span> <span class="nc">BigQueryPandasConnector</span><span class="p">(</span><span class="n">GbqConnector</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> This connector behaves identically to GbqConnector (from Pandas), except</span>
+<span class="sd"> that it allows the service to be injected, and disables a call to</span>
+<span class="sd"> self.get_credentials(). This allows Airflow to use BigQuery with Pandas</span>
+<span class="sd"> without forcing a three legged OAuth connection. Instead, we can inject</span>
+<span class="sd"> service account credentials into the binding.</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span>
+ <span class="n">service</span><span class="p">,</span>
+ <span class="n">reauth</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">dialect</span><span class="o">=</span><span class="s1">'legacy'</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryPandasConnector</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">project_id</span><span class="p">)</span>
+ <span class="n">gbq_check_google_client_version</span><span class="p">()</span>
+ <span class="n">gbq_test_google_api_imports</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">reauth</span> <span class="o">=</span> <span class="n">reauth</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">service</span> <span class="o">=</span> <span class="n">service</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">dialect</span> <span class="o">=</span> <span class="n">dialect</span>
+
+
+<span class="k">class</span> <span class="nc">BigQueryConnection</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> BigQuery does not have a notion of a persistent connection. Thus, these</span>
+<span class="sd"> objects are small stateless factories for cursors, which do all the real</span>
+<span class="sd"> work.</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_args</span> <span class="o">=</span> <span class="n">args</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_kwargs</span> <span class="o">=</span> <span class="n">kwargs</span>
+
+ <span class="k">def</span> <span class="nf">close</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">""" BigQueryConnection does not have anything to close. """</span>
+ <span class="k">pass</span>
+
+ <span class="k">def</span> <span class="nf">commit</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">""" BigQueryConnection does not support transactions. """</span>
+ <span class="k">pass</span>
+
+ <span class="k">def</span> <span class="nf">cursor</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">""" Return a new :py:class:`Cursor` object using the connection. """</span>
+ <span class="k">return</span> <span class="n">BigQueryCursor</span><span class="p">(</span><span class="o">*</span><span class="bp">self</span><span class="o">.</span><span class="n">_args</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">_kwargs</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">rollback</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span>
+ <span class="s2">"BigQueryConnection does not have transactions"</span><span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">BigQueryBaseCursor</span><span class="p">(</span><span class="n">LoggingMixin</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> The BigQuery base cursor contains helper methods to execute queries against</span>
+<span class="sd"> BigQuery. The methods can be used directly by operators, in cases where a</span>
+<span class="sd"> PEP 249 cursor isn't needed.</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">service</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span>
+ <span class="n">use_legacy_sql</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+ <span class="n">api_resource_configs</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">location</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">service</span> <span class="o">=</span> <span class="n">service</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span> <span class="o">=</span> <span class="n">use_legacy_sql</span>
+ <span class="k">if</span> <span class="n">api_resource_configs</span><span class="p">:</span>
+ <span class="n">_validate_value</span><span class="p">(</span><span class="s2">"api_resource_configs"</span><span class="p">,</span> <span class="n">api_resource_configs</span><span class="p">,</span> <span class="nb">dict</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">api_resource_configs</span> <span class="o">=</span> <span class="n">api_resource_configs</span> \
+ <span class="k">if</span> <span class="n">api_resource_configs</span> <span class="k">else</span> <span class="p">{}</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">location</span> <span class="o">=</span> <span class="n">location</span>
+
+ <span class="k">def</span> <span class="nf">create_empty_table</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span>
+ <span class="n">dataset_id</span><span class="p">,</span>
+ <span class="n">table_id</span><span class="p">,</span>
+ <span class="n">schema_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">time_partitioning</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">view</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Creates a new, empty table in the dataset.</span>
+<span class="sd"> To create a view, which is defined by a SQL query, parse a dictionary to 'view' kwarg</span>
+
+<span class="sd"> :param project_id: The project to create the table into.</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :param dataset_id: The dataset to create the table into.</span>
+<span class="sd"> :type dataset_id: str</span>
+<span class="sd"> :param table_id: The Name of the table to be created.</span>
+<span class="sd"> :type table_id: str</span>
+<span class="sd"> :param schema_fields: If set, the schema field list as defined here:</span>
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load.schema</span>
+<span class="sd"> :type schema_fields: list</span>
+<span class="sd"> :param labels: a dictionary containing labels for the table, passed to BigQuery</span>
+<span class="sd"> :type labels: dict</span>
+
+<span class="sd"> **Example**: ::</span>
+
+<span class="sd"> schema_fields=[{"name": "emp_name", "type": "STRING", "mode": "REQUIRED"},</span>
+<span class="sd"> {"name": "salary", "type": "INTEGER", "mode": "NULLABLE"}]</span>
+
+<span class="sd"> :param time_partitioning: configure optional time partitioning fields i.e.</span>
+<span class="sd"> partition by field, type and expiration as per API specifications.</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#timePartitioning</span>
+<span class="sd"> :type time_partitioning: dict</span>
+<span class="sd"> :param view: [Optional] A dictionary containing definition for the view.</span>
+<span class="sd"> If set, it will create a view instead of a table:</span>
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#view</span>
+<span class="sd"> :type view: dict</span>
+
+<span class="sd"> **Example**: ::</span>
+
+<span class="sd"> view = {</span>
+<span class="sd"> "query": "SELECT * FROM `test-project-id.test_dataset_id.test_table_prefix*` LIMIT 1000",</span>
+<span class="sd"> "useLegacySql": False</span>
+<span class="sd"> }</span>
+
+<span class="sd"> :return:</span>
+<span class="sd"> """</span>
+
+ <span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span> <span class="k">if</span> <span class="n">project_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
+
+ <span class="n">table_resource</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'tableReference'</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s1">'tableId'</span><span class="p">:</span> <span class="n">table_id</span>
+ <span class="p">}</span>
+ <span class="p">}</span>
+
+ <span class="k">if</span> <span class="n">schema_fields</span><span class="p">:</span>
+ <span class="n">table_resource</span><span class="p">[</span><span class="s1">'schema'</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'fields'</span><span class="p">:</span> <span class="n">schema_fields</span><span class="p">}</span>
+
+ <span class="k">if</span> <span class="n">time_partitioning</span><span class="p">:</span>
+ <span class="n">table_resource</span><span class="p">[</span><span class="s1">'timePartitioning'</span><span class="p">]</span> <span class="o">=</span> <span class="n">time_partitioning</span>
+
+ <span class="k">if</span> <span class="n">labels</span><span class="p">:</span>
+ <span class="n">table_resource</span><span class="p">[</span><span class="s1">'labels'</span><span class="p">]</span> <span class="o">=</span> <span class="n">labels</span>
+
+ <span class="k">if</span> <span class="n">view</span><span class="p">:</span>
+ <span class="n">table_resource</span><span class="p">[</span><span class="s1">'view'</span><span class="p">]</span> <span class="o">=</span> <span class="n">view</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Creating Table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+ <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Table created successfully: </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
+
+ <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'BigQuery job failed. Error was: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">err</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
+ <span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">create_external_table</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">external_project_dataset_table</span><span class="p">,</span>
+ <span class="n">schema_fields</span><span class="p">,</span>
+ <span class="n">source_uris</span><span class="p">,</span>
+ <span class="n">source_format</span><span class="o">=</span><span class="s1">'CSV'</span><span class="p">,</span>
+ <span class="n">autodetect</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">compression</span><span class="o">=</span><span class="s1">'NONE'</span><span class="p">,</span>
+ <span class="n">ignore_unknown_values</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">max_bad_records</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
+ <span class="n">skip_leading_rows</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
+ <span class="n">field_delimiter</span><span class="o">=</span><span class="s1">','</span><span class="p">,</span>
+ <span class="n">quote_character</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">allow_quoted_newlines</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">allow_jagged_rows</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">src_fmt_configs</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">labels</span><span class="o">=</span><span class="kc">None</span>
+ <span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Creates a new external table in the dataset with the data in Google</span>
+<span class="sd"> Cloud Storage. See here:</span>
+
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#resource</span>
+
+<span class="sd"> for more details about these parameters.</span>
+
+<span class="sd"> :param external_project_dataset_table:</span>
+<span class="sd"> The dotted (<project>.|<project>:)<dataset>.<table>($<partition>) BigQuery</span>
+<span class="sd"> table name to create external table.</span>
+<span class="sd"> If <project> is not included, project will be the</span>
+<span class="sd"> project defined in the connection json.</span>
+<span class="sd"> :type external_project_dataset_table: string</span>
+<span class="sd"> :param schema_fields: The schema field list as defined here:</span>
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#resource</span>
+<span class="sd"> :type schema_fields: list</span>
+<span class="sd"> :param source_uris: The source Google Cloud</span>
+<span class="sd"> Storage URI (e.g. gs://some-bucket/some-file.txt). A single wild</span>
+<span class="sd"> per-object name can be used.</span>
+<span class="sd"> :type source_uris: list</span>
+<span class="sd"> :param source_format: File format to export.</span>
+<span class="sd"> :type source_format: string</span>
+<span class="sd"> :param autodetect: Try to detect schema and format options automatically.</span>
+<span class="sd"> Any option specified explicitly will be honored.</span>
+<span class="sd"> :type autodetect: bool</span>
+<span class="sd"> :param compression: [Optional] The compression type of the data source.</span>
+<span class="sd"> Possible values include GZIP and NONE.</span>
+<span class="sd"> The default value is NONE.</span>
+<span class="sd"> This setting is ignored for Google Cloud Bigtable,</span>
+<span class="sd"> Google Cloud Datastore backups and Avro formats.</span>
+<span class="sd"> :type compression: string</span>
+<span class="sd"> :param ignore_unknown_values: [Optional] Indicates if BigQuery should allow</span>
+<span class="sd"> extra values that are not represented in the table schema.</span>
+<span class="sd"> If true, the extra values are ignored. If false, records with extra columns</span>
+<span class="sd"> are treated as bad records, and if there are too many bad records, an</span>
+<span class="sd"> invalid error is returned in the job result.</span>
+<span class="sd"> :type ignore_unknown_values: bool</span>
+<span class="sd"> :param max_bad_records: The maximum number of bad records that BigQuery can</span>
+<span class="sd"> ignore when running the job.</span>
+<span class="sd"> :type max_bad_records: int</span>
+<span class="sd"> :param skip_leading_rows: Number of rows to skip when loading from a CSV.</span>
+<span class="sd"> :type skip_leading_rows: int</span>
+<span class="sd"> :param field_delimiter: The delimiter to use when loading from a CSV.</span>
+<span class="sd"> :type field_delimiter: string</span>
+<span class="sd"> :param quote_character: The value that is used to quote data sections in a CSV</span>
+<span class="sd"> file.</span>
+<span class="sd"> :type quote_character: string</span>
+<span class="sd"> :param allow_quoted_newlines: Whether to allow quoted newlines (true) or not</span>
+<span class="sd"> (false).</span>
+<span class="sd"> :type allow_quoted_newlines: boolean</span>
+<span class="sd"> :param allow_jagged_rows: Accept rows that are missing trailing optional columns.</span>
+<span class="sd"> The missing values are treated as nulls. If false, records with missing</span>
+<span class="sd"> trailing columns are treated as bad records, and if there are too many bad</span>
+<span class="sd"> records, an invalid error is returned in the job result. Only applicable when</span>
+<span class="sd"> soure_format is CSV.</span>
+<span class="sd"> :type allow_jagged_rows: bool</span>
+<span class="sd"> :param src_fmt_configs: configure optional fields specific to the source format</span>
+<span class="sd"> :type src_fmt_configs: dict</span>
+<span class="sd"> :param labels: a dictionary containing labels for the table, passed to BigQuery</span>
+<span class="sd"> :type labels: dict</span>
+<span class="sd"> """</span>
+
+ <span class="k">if</span> <span class="n">src_fmt_configs</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">src_fmt_configs</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">external_table_id</span> <span class="o">=</span> \
+ <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">external_project_dataset_table</span><span class="p">,</span>
+ <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">var_name</span><span class="o">=</span><span class="s1">'external_project_dataset_table'</span><span class="p">)</span>
+
+ <span class="c1"># bigquery only allows certain source formats</span>
+ <span class="c1"># we check to make sure the passed source format is valid</span>
+ <span class="c1"># if it's not, we raise a ValueError</span>
+ <span class="c1"># Refer to this link for more details:</span>
+ <span class="c1"># https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#externalDataConfiguration.sourceFormat</span>
+
+ <span class="n">source_format</span> <span class="o">=</span> <span class="n">source_format</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
+ <span class="n">allowed_formats</span> <span class="o">=</span> <span class="p">[</span>
+ <span class="s2">"CSV"</span><span class="p">,</span> <span class="s2">"NEWLINE_DELIMITED_JSON"</span><span class="p">,</span> <span class="s2">"AVRO"</span><span class="p">,</span> <span class="s2">"GOOGLE_SHEETS"</span><span class="p">,</span>
+ <span class="s2">"DATASTORE_BACKUP"</span><span class="p">,</span> <span class="s2">"PARQUET"</span>
+ <span class="p">]</span>
+ <span class="k">if</span> <span class="n">source_format</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">allowed_formats</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"</span><span class="si">{0}</span><span class="s2"> is not a valid source format. "</span>
+ <span class="s2">"Please use one of the following types: </span><span class="si">{1}</span><span class="s2">"</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">source_format</span><span class="p">,</span> <span class="n">allowed_formats</span><span class="p">))</span>
+
+ <span class="n">compression</span> <span class="o">=</span> <span class="n">compression</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
+ <span class="n">allowed_compressions</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'NONE'</span><span class="p">,</span> <span class="s1">'GZIP'</span><span class="p">]</span>
+ <span class="k">if</span> <span class="n">compression</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">allowed_compressions</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"</span><span class="si">{0}</span><span class="s2"> is not a valid compression format. "</span>
+ <span class="s2">"Please use one of the following types: </span><span class="si">{1}</span><span class="s2">"</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">compression</span><span class="p">,</span> <span class="n">allowed_compressions</span><span class="p">))</span>
+
+ <span class="n">table_resource</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'externalDataConfiguration'</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s1">'autodetect'</span><span class="p">:</span> <span class="n">autodetect</span><span class="p">,</span>
+ <span class="s1">'sourceFormat'</span><span class="p">:</span> <span class="n">source_format</span><span class="p">,</span>
+ <span class="s1">'sourceUris'</span><span class="p">:</span> <span class="n">source_uris</span><span class="p">,</span>
+ <span class="s1">'compression'</span><span class="p">:</span> <span class="n">compression</span><span class="p">,</span>
+ <span class="s1">'ignoreUnknownValues'</span><span class="p">:</span> <span class="n">ignore_unknown_values</span>
+ <span class="p">},</span>
+ <span class="s1">'tableReference'</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s1">'projectId'</span><span class="p">:</span> <span class="n">project_id</span><span class="p">,</span>
+ <span class="s1">'datasetId'</span><span class="p">:</span> <span class="n">dataset_id</span><span class="p">,</span>
+ <span class="s1">'tableId'</span><span class="p">:</span> <span class="n">external_table_id</span><span class="p">,</span>
+ <span class="p">}</span>
+ <span class="p">}</span>
+
+ <span class="k">if</span> <span class="n">schema_fields</span><span class="p">:</span>
+ <span class="n">table_resource</span><span class="p">[</span><span class="s1">'externalDataConfiguration'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">({</span>
+ <span class="s1">'schema'</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s1">'fields'</span><span class="p">:</span> <span class="n">schema_fields</span>
+ <span class="p">}</span>
+ <span class="p">})</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Creating external table: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">external_project_dataset_table</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">max_bad_records</span><span class="p">:</span>
+ <span class="n">table_resource</span><span class="p">[</span><span class="s1">'externalDataConfiguration'</span><span class="p">][</span><span class="s1">'maxBadRecords'</span><span class="p">]</span> <span class="o">=</span> <span class="n">max_bad_records</span>
+
+ <span class="c1"># if following fields are not specified in src_fmt_configs,</span>
+ <span class="c1"># honor the top-level params for backward-compatibility</span>
+ <span class="k">if</span> <span class="s1">'skipLeadingRows'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
+ <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">'skipLeadingRows'</span><span class="p">]</span> <span class="o">=</span> <span class="n">skip_leading_rows</span>
+ <span class="k">if</span> <span class="s1">'fieldDelimiter'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
+ <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">'fieldDelimiter'</span><span class="p">]</span> <span class="o">=</span> <span class="n">field_delimiter</span>
+ <span class="k">if</span> <span class="s1">'quote_character'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
+ <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">'quote'</span><span class="p">]</span> <span class="o">=</span> <span class="n">quote_character</span>
+ <span class="k">if</span> <span class="s1">'allowQuotedNewlines'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
+ <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">'allowQuotedNewlines'</span><span class="p">]</span> <span class="o">=</span> <span class="n">allow_quoted_newlines</span>
+ <span class="k">if</span> <span class="s1">'allowJaggedRows'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
+ <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">'allowJaggedRows'</span><span class="p">]</span> <span class="o">=</span> <span class="n">allow_jagged_rows</span>
+
+ <span class="n">src_fmt_to_param_mapping</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'CSV'</span><span class="p">:</span> <span class="s1">'csvOptions'</span><span class="p">,</span>
+ <span class="s1">'GOOGLE_SHEETS'</span><span class="p">:</span> <span class="s1">'googleSheetsOptions'</span>
+ <span class="p">}</span>
+
+ <span class="n">src_fmt_to_configs_mapping</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'csvOptions'</span><span class="p">:</span> <span class="p">[</span>
+ <span class="s1">'allowJaggedRows'</span><span class="p">,</span> <span class="s1">'allowQuotedNewlines'</span><span class="p">,</span>
+ <span class="s1">'fieldDelimiter'</span><span class="p">,</span> <span class="s1">'skipLeadingRows'</span><span class="p">,</span>
+ <span class="s1">'quote'</span>
+ <span class="p">],</span>
+ <span class="s1">'googleSheetsOptions'</span><span class="p">:</span> <span class="p">[</span><span class="s1">'skipLeadingRows'</span><span class="p">]</span>
+ <span class="p">}</span>
+
+ <span class="k">if</span> <span class="n">source_format</span> <span class="ow">in</span> <span class="n">src_fmt_to_param_mapping</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
+
+ <span class="n">valid_configs</span> <span class="o">=</span> <span class="n">src_fmt_to_configs_mapping</span><span class="p">[</span>
+ <span class="n">src_fmt_to_param_mapping</span><span class="p">[</span><span class="n">source_format</span><span class="p">]</span>
+ <span class="p">]</span>
+
+ <span class="n">src_fmt_configs</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="n">k</span><span class="p">:</span> <span class="n">v</span>
+ <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="k">if</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">valid_configs</span>
+ <span class="p">}</span>
+
+ <span class="n">table_resource</span><span class="p">[</span><span class="s1">'externalDataConfiguration'</span><span class="p">][</span><span class="n">src_fmt_to_param_mapping</span><span class="p">[</span>
+ <span class="n">source_format</span><span class="p">]]</span> <span class="o">=</span> <span class="n">src_fmt_configs</span>
+
+ <span class="k">if</span> <span class="n">labels</span><span class="p">:</span>
+ <span class="n">table_resource</span><span class="p">[</span><span class="s1">'labels'</span><span class="p">]</span> <span class="o">=</span> <span class="n">labels</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+ <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'External table created successfully: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">external_project_dataset_table</span><span class="p">)</span>
+
+ <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
+ <span class="s1">'BigQuery job failed. Error was: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">err</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
+ <span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">run_query</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">bql</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">sql</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">destination_dataset_table</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">write_disposition</span><span class="o">=</span><span class="s1">'WRITE_EMPTY'</span><span class="p">,</span>
+ <span class="n">allow_large_results</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">flatten_results</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">udf_config</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">use_legacy_sql</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">maximum_billing_tier</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">maximum_bytes_billed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">create_disposition</span><span class="o">=</span><span class="s1">'CREATE_IF_NEEDED'</span><span class="p">,</span>
+ <span class="n">query_params</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">schema_update_options</span><span class="o">=</span><span class="p">(),</span>
+ <span class="n">priority</span><span class="o">=</span><span class="s1">'INTERACTIVE'</span><span class="p">,</span>
+ <span class="n">time_partitioning</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">api_resource_configs</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">cluster_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">location</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Executes a BigQuery SQL query. Optionally persists results in a BigQuery</span>
+<span class="sd"> table. See here:</span>
+
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/v2/jobs</span>
+
+<span class="sd"> For more details about these parameters.</span>
+
+<span class="sd"> :param bql: (Deprecated. Use `sql` parameter instead) The BigQuery SQL</span>
+<span class="sd"> to execute.</span>
+<span class="sd"> :type bql: string</span>
+<span class="sd"> :param sql: The BigQuery SQL to execute.</span>
+<span class="sd"> :type sql: string</span>
+<span class="sd"> :param destination_dataset_table: The dotted <dataset>.<table></span>
+<span class="sd"> BigQuery table to save the query results.</span>
+<span class="sd"> :type destination_dataset_table: string</span>
+<span class="sd"> :param write_disposition: What to do if the table already exists in</span>
+<span class="sd"> BigQuery.</span>
+<span class="sd"> :type write_disposition: string</span>
+<span class="sd"> :param allow_large_results: Whether to allow large results.</span>
+<span class="sd"> :type allow_large_results: boolean</span>
+<span class="sd"> :param flatten_results: If true and query uses legacy SQL dialect, flattens</span>
+<span class="sd"> all nested and repeated fields in the query results. ``allowLargeResults``</span>
+<span class="sd"> must be true if this is set to false. For standard SQL queries, this</span>
+<span class="sd"> flag is ignored and results are never flattened.</span>
+<span class="sd"> :type flatten_results: boolean</span>
+<span class="sd"> :param udf_config: The User Defined Function configuration for the query.</span>
+<span class="sd"> See https://cloud.google.com/bigquery/user-defined-functions for details.</span>
+<span class="sd"> :type udf_config: list</span>
+<span class="sd"> :param use_legacy_sql: Whether to use legacy SQL (true) or standard SQL (false).</span>
+<span class="sd"> If `None`, defaults to `self.use_legacy_sql`.</span>
+<span class="sd"> :type use_legacy_sql: boolean</span>
+<span class="sd"> :param api_resource_configs: a dictionary that contain params</span>
+<span class="sd"> 'configuration' applied for Google BigQuery Jobs API:</span>
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs</span>
+<span class="sd"> for example, {'query': {'useQueryCache': False}}. You could use it</span>
+<span class="sd"> if you need to provide some params that are not supported by the</span>
+<span class="sd"> BigQueryHook like args.</span>
+<span class="sd"> :type api_resource_configs: dict</span>
+<span class="sd"> :param maximum_billing_tier: Positive integer that serves as a</span>
+<span class="sd"> multiplier of the basic price.</span>
+<span class="sd"> :type maximum_billing_tier: integer</span>
+<span class="sd"> :param maximum_bytes_billed: Limits the bytes billed for this job.</span>
+<span class="sd"> Queries that will have bytes billed beyond this limit will fail</span>
+<span class="sd"> (without incurring a charge). If unspecified, this will be</span>
+<span class="sd"> set to your project default.</span>
+<span class="sd"> :type maximum_bytes_billed: float</span>
+<span class="sd"> :param create_disposition: Specifies whether the job is allowed to</span>
+<span class="sd"> create new tables.</span>
+<span class="sd"> :type create_disposition: string</span>
+<span class="sd"> :param query_params a dictionary containing query parameter types and</span>
+<span class="sd"> values, passed to BigQuery</span>
+<span class="sd"> :type query_params: dict</span>
+<span class="sd"> :param labels a dictionary containing labels for the job/query,</span>
+<span class="sd"> passed to BigQuery</span>
+<span class="sd"> :type labels: dict</span>
+<span class="sd"> :param schema_update_options: Allows the schema of the destination</span>
+<span class="sd"> table to be updated as a side effect of the query job.</span>
+<span class="sd"> :type schema_update_options: tuple</span>
+<span class="sd"> :param priority: Specifies a priority for the query.</span>
+<span class="sd"> Possible values include INTERACTIVE and BATCH.</span>
+<span class="sd"> The default value is INTERACTIVE.</span>
+<span class="sd"> :type priority: string</span>
+<span class="sd"> :param time_partitioning: configure optional time partitioning fields i.e.</span>
+<span class="sd"> partition by field, type and expiration as per API specifications.</span>
+<span class="sd"> :type time_partitioning: dict</span>
+<span class="sd"> :param cluster_fields: Request that the result of this query be stored sorted</span>
+<span class="sd"> by one or more columns. This is only available in combination with</span>
+<span class="sd"> time_partitioning. The order of columns given determines the sort order.</span>
+<span class="sd"> :type cluster_fields: list of str</span>
+<span class="sd"> :param location: The geographic location of the job. Required except for</span>
+<span class="sd"> US and EU. See details at</span>
+<span class="sd"> https://cloud.google.com/bigquery/docs/locations#specifying_your_location</span>
+<span class="sd"> :type location: str</span>
+<span class="sd"> """</span>
+
+ <span class="k">if</span> <span class="n">time_partitioning</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">time_partitioning</span> <span class="o">=</span> <span class="p">{}</span>
+
+ <span class="k">if</span> <span class="n">location</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">location</span> <span class="o">=</span> <span class="n">location</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">api_resource_configs</span><span class="p">:</span>
+ <span class="n">api_resource_configs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">api_resource_configs</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">_validate_value</span><span class="p">(</span><span class="s1">'api_resource_configs'</span><span class="p">,</span>
+ <span class="n">api_resource_configs</span><span class="p">,</span> <span class="nb">dict</span><span class="p">)</span>
+ <span class="n">configuration</span> <span class="o">=</span> <span class="n">deepcopy</span><span class="p">(</span><span class="n">api_resource_configs</span><span class="p">)</span>
+ <span class="k">if</span> <span class="s1">'query'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">configuration</span><span class="p">:</span>
+ <span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
+
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">_validate_value</span><span class="p">(</span><span class="s2">"api_resource_configs['query']"</span><span class="p">,</span>
+ <span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">],</span> <span class="nb">dict</span><span class="p">)</span>
+
+ <span class="n">sql</span> <span class="o">=</span> <span class="n">bql</span> <span class="k">if</span> <span class="n">sql</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">sql</span>
+
+ <span class="c1"># TODO remove `bql` in Airflow 2.0 - Jira: [AIRFLOW-2513]</span>
+ <span class="k">if</span> <span class="n">bql</span><span class="p">:</span>
+ <span class="kn">import</span> <span class="nn">warnings</span>
+ <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s1">'Deprecated parameter `bql` used in '</span>
+ <span class="s1">'`BigQueryBaseCursor.run_query` '</span>
+ <span class="s1">'Use `sql` parameter instead to pass the sql to be '</span>
+ <span class="s1">'executed. `bql` parameter is deprecated and '</span>
+ <span class="s1">'will be removed in a future version of '</span>
+ <span class="s1">'Airflow.'</span><span class="p">,</span>
+ <span class="n">category</span><span class="o">=</span><span class="ne">DeprecationWarning</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">sql</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'query'</span><span class="p">,</span> <span class="kc">None</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">'`BigQueryBaseCursor.run_query` '</span>
+ <span class="s1">'missing 1 required positional argument: `sql`'</span><span class="p">)</span>
+
+ <span class="c1"># BigQuery also allows you to define how you want a table's schema to change</span>
+ <span class="c1"># as a side effect of a query job</span>
+ <span class="c1"># for more details:</span>
+ <span class="c1"># https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.query.schemaUpdateOptions</span>
+
+ <span class="n">allowed_schema_update_options</span> <span class="o">=</span> <span class="p">[</span>
+ <span class="s1">'ALLOW_FIELD_ADDITION'</span><span class="p">,</span> <span class="s2">"ALLOW_FIELD_RELAXATION"</span>
+ <span class="p">]</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="nb">set</span><span class="p">(</span><span class="n">allowed_schema_update_options</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">issuperset</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">)):</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"</span><span class="si">{0}</span><span class="s2"> contains invalid schema update options. "</span>
+ <span class="s2">"Please only use one or more of the following "</span>
+ <span class="s2">"options: </span><span class="si">{1}</span><span class="s2">"</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">,</span>
+ <span class="n">allowed_schema_update_options</span><span class="p">))</span>
+
+ <span class="k">if</span> <span class="n">schema_update_options</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">write_disposition</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">"WRITE_APPEND"</span><span class="p">,</span> <span class="s2">"WRITE_TRUNCATE"</span><span class="p">]:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"schema_update_options is only "</span>
+ <span class="s2">"allowed if write_disposition is "</span>
+ <span class="s2">"'WRITE_APPEND' or 'WRITE_TRUNCATE'."</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">destination_dataset_table</span><span class="p">:</span>
+ <span class="n">destination_project</span><span class="p">,</span> <span class="n">destination_dataset</span><span class="p">,</span> <span class="n">destination_table</span> <span class="o">=</span> \
+ <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">destination_dataset_table</span><span class="p">,</span>
+ <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">)</span>
+
+ <span class="n">destination_dataset_table</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'projectId'</span><span class="p">:</span> <span class="n">destination_project</span><span class="p">,</span>
+ <span class="s1">'datasetId'</span><span class="p">:</span> <span class="n">destination_dataset</span><span class="p">,</span>
+ <span class="s1">'tableId'</span><span class="p">:</span> <span class="n">destination_table</span><span class="p">,</span>
+ <span class="p">}</span>
+
+ <span class="k">if</span> <span class="n">cluster_fields</span><span class="p">:</span>
+ <span class="n">cluster_fields</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'fields'</span><span class="p">:</span> <span class="n">cluster_fields</span><span class="p">}</span>
+
+ <span class="n">query_param_list</span> <span class="o">=</span> <span class="p">[</span>
+ <span class="p">(</span><span class="n">sql</span><span class="p">,</span> <span class="s1">'query'</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="n">six</span><span class="o">.</span><span class="n">string_types</span><span class="p">),</span>
+ <span class="p">(</span><span class="n">priority</span><span class="p">,</span> <span class="s1">'priority'</span><span class="p">,</span> <span class="s1">'INTERACTIVE'</span><span class="p">,</span> <span class="n">six</span><span class="o">.</span><span class="n">string_types</span><span class="p">),</span>
+ <span class="p">(</span><span class="n">use_legacy_sql</span><span class="p">,</span> <span class="s1">'useLegacySql'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span><span class="p">,</span> <span class="nb">bool</span><span class="p">),</span>
+ <span class="p">(</span><span class="n">query_params</span><span class="p">,</span> <span class="s1">'queryParameters'</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="nb">dict</span><span class="p">),</span>
+ <span class="p">(</span><span class="n">udf_config</span><span class="p">,</span> <span class="s1">'userDefinedFunctionResources'</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="nb">list</span><span class="p">),</span>
+ <span class="p">(</span><span class="n">maximum_billing_tier</span><span class="p">,</span> <span class="s1">'maximumBillingTier'</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="nb">int</span><span class="p">),</span>
+ <span class="p">(</span><span class="n">maximum_bytes_billed</span><span class="p">,</span> <span class="s1">'maximumBytesBilled'</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="nb">float</span><span class="p">),</span>
+ <span class="p">(</span><span class="n">time_partitioning</span><span class="p">,</span> <span class="s1">'timePartitioning'</span><span class="p">,</span> <span class="p">{},</span> <span class="nb">dict</span><span class="p">),</span>
+ <span class="p">(</span><span class="n">schema_update_options</span><span class="p">,</span> <span class="s1">'schemaUpdateOptions'</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">),</span>
+ <span class="p">(</span><span class="n">destination_dataset_table</span><span class="p">,</span> <span class="s1">'destinationTable'</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="nb">dict</span><span class="p">),</span>
+ <span class="p">(</span><span class="n">cluster_fields</span><span class="p">,</span> <span class="s1">'clustering'</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="nb">dict</span><span class="p">),</span>
+ <span class="p">]</span>
+
+ <span class="k">for</span> <span class="n">param_tuple</span> <span class="ow">in</span> <span class="n">query_param_list</span><span class="p">:</span>
+
+ <span class="n">param</span><span class="p">,</span> <span class="n">param_name</span><span class="p">,</span> <span class="n">param_default</span><span class="p">,</span> <span class="n">param_type</span> <span class="o">=</span> <span class="n">param_tuple</span>
+
+ <span class="k">if</span> <span class="n">param_name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">]</span> <span class="ow">and</span> <span class="n">param</span> <span class="ow">in</span> <span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="p">{},</span> <span class="p">()]:</span>
+ <span class="k">if</span> <span class="n">param_name</span> <span class="o">==</span> <span class="s1">'timePartitioning'</span><span class="p">:</span>
+ <span class="n">param_default</span> <span class="o">=</span> <span class="n">_cleanse_time_partitioning</span><span class="p">(</span>
+ <span class="n">destination_dataset_table</span><span class="p">,</span> <span class="n">time_partitioning</span><span class="p">)</span>
+ <span class="n">param</span> <span class="o">=</span> <span class="n">param_default</span>
+
+ <span class="k">if</span> <span class="n">param</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="p">{},</span> <span class="p">()]:</span>
+ <span class="n">_api_resource_configs_duplication_check</span><span class="p">(</span>
+ <span class="n">param_name</span><span class="p">,</span> <span class="n">param</span><span class="p">,</span> <span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">])</span>
+
+ <span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">][</span><span class="n">param_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">param</span>
+
+ <span class="c1"># check valid type of provided param,</span>
+ <span class="c1"># it last step because we can get param from 2 sources,</span>
+ <span class="c1"># and first of all need to find it</span>
+
+ <span class="n">_validate_value</span><span class="p">(</span><span class="n">param_name</span><span class="p">,</span> <span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">][</span><span class="n">param_name</span><span class="p">],</span>
+ <span class="n">param_type</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">param_name</span> <span class="o">==</span> <span class="s1">'schemaUpdateOptions'</span> <span class="ow">and</span> <span class="n">param</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Adding experimental 'schemaUpdateOptions': "</span>
+ <span class="s2">"</span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">))</span>
+
+ <span class="k">if</span> <span class="n">param_name</span> <span class="o">==</span> <span class="s1">'destinationTable'</span><span class="p">:</span>
+ <span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'projectId'</span><span class="p">,</span> <span class="s1">'datasetId'</span><span class="p">,</span> <span class="s1">'tableId'</span><span class="p">]:</span>
+ <span class="k">if</span> <span class="n">key</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">][</span><span class="s1">'destinationTable'</span><span class="p">]:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+ <span class="s2">"Not correct 'destinationTable' in "</span>
+ <span class="s2">"api_resource_configs. 'destinationTable' "</span>
+ <span class="s2">"must be a dict with {'projectId':'', "</span>
+ <span class="s2">"'datasetId':'', 'tableId':''}"</span><span class="p">)</span>
+
+ <span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">({</span>
+ <span class="s1">'allowLargeResults'</span><span class="p">:</span> <span class="n">allow_large_results</span><span class="p">,</span>
+ <span class="s1">'flattenResults'</span><span class="p">:</span> <span class="n">flatten_results</span><span class="p">,</span>
+ <span class="s1">'writeDisposition'</span><span class="p">:</span> <span class="n">write_disposition</span><span class="p">,</span>
+ <span class="s1">'createDisposition'</span><span class="p">:</span> <span class="n">create_disposition</span><span class="p">,</span>
+ <span class="p">})</span>
+
+ <span class="k">if</span> <span class="s1">'useLegacySql'</span> <span class="ow">in</span> <span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">]</span> <span class="ow">and</span> \
+ <span class="s1">'queryParameters'</span> <span class="ow">in</span> <span class="n">configuration</span><span class="p">[</span><span class="s1">'query'</span><span class="p">]:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Query parameters are not allowed "</span>
+ <span class="s2">"when using legacy SQL"</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">labels</span><span class="p">:</span>
+ <span class="n">_api_resource_configs_duplication_check</span><span class="p">(</span>
+ <span class="s1">'labels'</span><span class="p">,</span> <span class="n">labels</span><span class="p">,</span> <span class="n">configuration</span><span class="p">)</span>
+ <span class="n">configuration</span><span class="p">[</span><span class="s1">'labels'</span><span class="p">]</span> <span class="o">=</span> <span class="n">labels</span>
+
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_with_configuration</span><span class="p">(</span><span class="n">configuration</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">run_extract</span><span class="p">(</span> <span class="c1"># noqa</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">source_project_dataset_table</span><span class="p">,</span>
+ <span class="n">destination_cloud_storage_uris</span><span class="p">,</span>
+ <span class="n">compression</span><span class="o">=</span><span class="s1">'NONE'</span><span class="p">,</span>
+ <span class="n">export_format</span><span class="o">=</span><span class="s1">'CSV'</span><span class="p">,</span>
+ <span class="n">field_delimiter</span><span class="o">=</span><span class="s1">','</span><span class="p">,</span>
+ <span class="n">print_header</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+ <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Executes a BigQuery extract command to copy data from BigQuery to</span>
+<span class="sd"> Google Cloud Storage. See here:</span>
+
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/v2/jobs</span>
+
+<span class="sd"> For more details about these parameters.</span>
+
+<span class="sd"> :param source_project_dataset_table: The dotted <dataset>.<table></span>
+<span class="sd"> BigQuery table to use as the source data.</span>
+<span class="sd"> :type source_project_dataset_table: string</span>
+<span class="sd"> :param destination_cloud_storage_uris: The destination Google Cloud</span>
+<span class="sd"> Storage URI (e.g. gs://some-bucket/some-file.txt). Follows</span>
+<span class="sd"> convention defined here:</span>
+<span class="sd"> https://cloud.google.com/bigquery/exporting-data-from-bigquery#exportingmultiple</span>
+<span class="sd"> :type destination_cloud_storage_uris: list</span>
+<span class="sd"> :param compression: Type of compression to use.</span>
+<span class="sd"> :type compression: string</span>
+<span class="sd"> :param export_format: File format to export.</span>
+<span class="sd"> :type export_format: string</span>
+<span class="sd"> :param field_delimiter: The delimiter to use when extracting to a CSV.</span>
+<span class="sd"> :type field_delimiter: string</span>
+<span class="sd"> :param print_header: Whether to print a header for a CSV file extract.</span>
+<span class="sd"> :type print_header: boolean</span>
+<span class="sd"> :param labels: a dictionary containing labels for the job/query,</span>
+<span class="sd"> passed to BigQuery</span>
+<span class="sd"> :type labels: dict</span>
+<span class="sd"> """</span>
+
+ <span class="n">source_project</span><span class="p">,</span> <span class="n">source_dataset</span><span class="p">,</span> <span class="n">source_table</span> <span class="o">=</span> \
+ <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">source_project_dataset_table</span><span class="p">,</span>
+ <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">var_name</span><span class="o">=</span><span class="s1">'source_project_dataset_table'</span><span class="p">)</span>
+
+ <span class="n">configuration</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'extract'</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s1">'sourceTable'</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s1">'projectId'</span><span class="p">:</span> <span class="n">source_project</span><span class="p">,</span>
+ <span class="s1">'datasetId'</span><span class="p">:</span> <span class="n">source_dataset</span><span class="p">,</span>
+ <span class="s1">'tableId'</span><span class="p">:</span> <span class="n">source_table</span><span class="p">,</span>
+ <span class="p">},</span>
+ <span class="s1">'compression'</span><span class="p">:</span> <span class="n">compression</span><span class="p">,</span>
+ <span class="s1">'destinationUris'</span><span class="p">:</span> <span class="n">destination_cloud_storage_uris</span><span class="p">,</span>
+ <span class="s1">'destinationFormat'</span><span class="p">:</span> <span class="n">export_format</span><span class="p">,</span>
+ <span class="p">}</span>
+ <span class="p">}</span>
+
+ <span class="k">if</span> <span class="n">labels</span><span class="p">:</span>
+ <span class="n">configuration</span><span class="p">[</span><span class="s1">'labels'</span><span class="p">]</span> <span class="o">=</span> <span class="n">labels</span>
+
+ <span class="k">if</span> <span class="n">export_format</span> <span class="o">==</span> <span class="s1">'CSV'</span><span class="p">:</span>
+ <span class="c1"># Only set fieldDelimiter and printHeader fields if using CSV.</span>
+ <span class="c1"># Google does not like it if you set these fields for other export</span>
+ <span class="c1"># formats.</span>
+ <span class="n">configuration</span><span class="p">[</span><span class="s1">'extract'</span><span class="p">][</span><span class="s1">'fieldDelimiter'</span><span class="p">]</span> <span class="o">=</span> <span class="n">field_delimiter</span>
+ <span class="n">configuration</span><span class="p">[</span><span class="s1">'extract'</span><span class="p">][</span><span class="s1">'printHeader'</span><span class="p">]</span> <span class="o">=</span> <span class="n">print_header</span>
+
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_with_configuration</span><span class="p">(</span><span class="n">configuration</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">run_copy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">source_project_dataset_tables</span><span class="p">,</span>
+ <span class="n">destination_project_dataset_table</span><span class="p">,</span>
+ <span class="n">write_disposition</span><span class="o">=</span><span class="s1">'WRITE_EMPTY'</span><span class="p">,</span>
+ <span class="n">create_disposition</span><span class="o">=</span><span class="s1">'CREATE_IF_NEEDED'</span><span class="p">,</span>
+ <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Executes a BigQuery copy command to copy data from one BigQuery table</span>
+<span class="sd"> to another. See here:</span>
+
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.copy</span>
+
+<span class="sd"> For more details about these parameters.</span>
+
+<span class="sd"> :param source_project_dataset_tables: One or more dotted</span>
+<span class="sd"> (project:|project.)<dataset>.<table></span>
+<span class="sd"> BigQuery tables to use as the source data. Use a list if there are</span>
+<span class="sd"> multiple source tables.</span>
+<span class="sd"> If <project> is not included, project will be the project defined</span>
+<span class="sd"> in the connection json.</span>
+<span class="sd"> :type source_project_dataset_tables: list|string</span>
+<span class="sd"> :param destination_project_dataset_table: The destination BigQuery</span>
+<span class="sd"> table. Format is: (project:|project.)<dataset>.<table></span>
+<span class="sd"> :type destination_project_dataset_table: string</span>
+<span class="sd"> :param write_disposition: The write disposition if the table already exists.</span>
+<span class="sd"> :type write_disposition: string</span>
+<span class="sd"> :param create_disposition: The create disposition if the table doesn't exist.</span>
+<span class="sd"> :type create_disposition: string</span>
+<span class="sd"> :param labels a dictionary containing labels for the job/query,</span>
+<span class="sd"> passed to BigQuery</span>
+<span class="sd"> :type labels: dict</span>
+<span class="sd"> """</span>
+ <span class="n">source_project_dataset_tables</span> <span class="o">=</span> <span class="p">([</span>
+ <span class="n">source_project_dataset_tables</span>
+ <span class="p">]</span> <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">source_project_dataset_tables</span><span class="p">,</span> <span class="nb">list</span><span class="p">)</span> <span class="k">else</span>
+ <span class="n">source_project_dataset_tables</span><span class="p">)</span>
+
+ <span class="n">source_project_dataset_tables_fixup</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="k">for</span> <span class="n">source_project_dataset_table</span> <span class="ow">in</span> <span class="n">source_project_dataset_tables</span><span class="p">:</span>
+ <span class="n">source_project</span><span class="p">,</span> <span class="n">source_dataset</span><span class="p">,</span> <span class="n">source_table</span> <span class="o">=</span> \
+ <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">source_project_dataset_table</span><span class="p">,</span>
+ <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">var_name</span><span class="o">=</span><span class="s1">'source_project_dataset_table'</span><span class="p">)</span>
+ <span class="n">source_project_dataset_tables_fixup</span><span class="o">.</span><span class="n">append</span><span class="p">({</span>
+ <span class="s1">'projectId'</span><span class="p">:</span>
+ <span class="n">source_project</span><span class="p">,</span>
+ <span class="s1">'datasetId'</span><span class="p">:</span>
+ <span class="n">source_dataset</span><span class="p">,</span>
+ <span class="s1">'tableId'</span><span class="p">:</span>
+ <span class="n">source_table</span>
+ <span class="p">})</span>
+
+ <span class="n">destination_project</span><span class="p">,</span> <span class="n">destination_dataset</span><span class="p">,</span> <span class="n">destination_table</span> <span class="o">=</span> \
+ <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">destination_project_dataset_table</span><span class="p">,</span>
+ <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">)</span>
+ <span class="n">configuration</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'copy'</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s1">'createDisposition'</span><span class="p">:</span> <span class="n">create_disposition</span><span class="p">,</span>
+ <span class="s1">'writeDisposition'</span><span class="p">:</span> <span class="n">write_disposition</span><span class="p">,</span>
+ <span class="s1">'sourceTables'</span><span class="p">:</span> <span class="n">source_project_dataset_tables_fixup</span><span class="p">,</span>
+ <span class="s1">'destinationTable'</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s1">'projectId'</span><span class="p">:</span> <span class="n">destination_project</span><span class="p">,</span>
+ <span class="s1">'datasetId'</span><span class="p">:</span> <span class="n">destination_dataset</span><span class="p">,</span>
+ <span class="s1">'tableId'</span><span class="p">:</span> <span class="n">destination_table</span>
+ <span class="p">}</span>
+ <span class="p">}</span>
+ <span class="p">}</span>
+
+ <span class="k">if</span> <span class="n">labels</span><span class="p">:</span>
+ <span class="n">configuration</span><span class="p">[</span><span class="s1">'labels'</span><span class="p">]</span> <span class="o">=</span> <span class="n">labels</span>
+
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_with_configuration</span><span class="p">(</span><span class="n">configuration</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">run_load</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">destination_project_dataset_table</span><span class="p">,</span>
+ <span class="n">source_uris</span><span class="p">,</span>
+ <span class="n">schema_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">source_format</span><span class="o">=</span><span class="s1">'CSV'</span><span class="p">,</span>
+ <span class="n">create_disposition</span><span class="o">=</span><span class="s1">'CREATE_IF_NEEDED'</span><span class="p">,</span>
+ <span class="n">skip_leading_rows</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
+ <span class="n">write_disposition</span><span class="o">=</span><span class="s1">'WRITE_EMPTY'</span><span class="p">,</span>
+ <span class="n">field_delimiter</span><span class="o">=</span><span class="s1">','</span><span class="p">,</span>
+ <span class="n">max_bad_records</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
+ <span class="n">quote_character</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">ignore_unknown_values</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">allow_quoted_newlines</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">allow_jagged_rows</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">schema_update_options</span><span class="o">=</span><span class="p">(),</span>
+ <span class="n">src_fmt_configs</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">time_partitioning</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">cluster_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">autodetect</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Executes a BigQuery load command to load data from Google Cloud Storage</span>
+<span class="sd"> to BigQuery. See here:</span>
+
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/v2/jobs</span>
+
+<span class="sd"> For more details about these parameters.</span>
+
+<span class="sd"> :param destination_project_dataset_table:</span>
+<span class="sd"> The dotted (<project>.|<project>:)<dataset>.<table>($<partition>) BigQuery</span>
+<span class="sd"> table to load data into. If <project> is not included, project will be the</span>
+<span class="sd"> project defined in the connection json. If a partition is specified the</span>
+<span class="sd"> operator will automatically append the data, create a new partition or create</span>
+<span class="sd"> a new DAY partitioned table.</span>
+<span class="sd"> :type destination_project_dataset_table: string</span>
+<span class="sd"> :param schema_fields: The schema field list as defined here:</span>
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load</span>
+<span class="sd"> Required if autodetect=False; optional if autodetect=True.</span>
+<span class="sd"> :type schema_fields: list</span>
+<span class="sd"> :param autodetect: Attempt to autodetect the schema for CSV and JSON</span>
+<span class="sd"> source files.</span>
+<span class="sd"> :type autodetect: bool</span>
+<span class="sd"> :param source_uris: The source Google Cloud</span>
+<span class="sd"> Storage URI (e.g. gs://some-bucket/some-file.txt). A single wild</span>
+<span class="sd"> per-object name can be used.</span>
+<span class="sd"> :type source_uris: list</span>
+<span class="sd"> :param source_format: File format to export.</span>
+<span class="sd"> :type source_format: string</span>
+<span class="sd"> :param create_disposition: The create disposition if the table doesn't exist.</span>
+<span class="sd"> :type create_disposition: string</span>
+<span class="sd"> :param skip_leading_rows: Number of rows to skip when loading from a CSV.</span>
+<span class="sd"> :type skip_leading_rows: int</span>
+<span class="sd"> :param write_disposition: The write disposition if the table already exists.</span>
+<span class="sd"> :type write_disposition: string</span>
+<span class="sd"> :param field_delimiter: The delimiter to use when loading from a CSV.</span>
+<span class="sd"> :type field_delimiter: string</span>
+<span class="sd"> :param max_bad_records: The maximum number of bad records that BigQuery can</span>
+<span class="sd"> ignore when running the job.</span>
+<span class="sd"> :type max_bad_records: int</span>
+<span class="sd"> :param quote_character: The value that is used to quote data sections in a CSV</span>
+<span class="sd"> file.</span>
+<span class="sd"> :type quote_character: string</span>
+<span class="sd"> :param ignore_unknown_values: [Optional] Indicates if BigQuery should allow</span>
+<span class="sd"> extra values that are not represented in the table schema.</span>
+<span class="sd"> If true, the extra values are ignored. If false, records with extra columns</span>
+<span class="sd"> are treated as bad records, and if there are too many bad records, an</span>
+<span class="sd"> invalid error is returned in the job result.</span>
+<span class="sd"> :type ignore_unknown_values: bool</span>
+<span class="sd"> :param allow_quoted_newlines: Whether to allow quoted newlines (true) or not</span>
+<span class="sd"> (false).</span>
+<span class="sd"> :type allow_quoted_newlines: boolean</span>
+<span class="sd"> :param allow_jagged_rows: Accept rows that are missing trailing optional columns.</span>
+<span class="sd"> The missing values are treated as nulls. If false, records with missing</span>
+<span class="sd"> trailing columns are treated as bad records, and if there are too many bad</span>
+<span class="sd"> records, an invalid error is returned in the job result. Only applicable when</span>
+<span class="sd"> soure_format is CSV.</span>
+<span class="sd"> :type allow_jagged_rows: bool</span>
+<span class="sd"> :param schema_update_options: Allows the schema of the desitination</span>
+<span class="sd"> table to be updated as a side effect of the load job.</span>
+<span class="sd"> :type schema_update_options: tuple</span>
+<span class="sd"> :param src_fmt_configs: configure optional fields specific to the source format</span>
+<span class="sd"> :type src_fmt_configs: dict</span>
+<span class="sd"> :param time_partitioning: configure optional time partitioning fields i.e.</span>
+<span class="sd"> partition by field, type and expiration as per API specifications.</span>
+<span class="sd"> :type time_partitioning: dict</span>
+<span class="sd"> :param cluster_fields: Request that the result of this load be stored sorted</span>
+<span class="sd"> by one or more columns. This is only available in combination with</span>
+<span class="sd"> time_partitioning. The order of columns given determines the sort order.</span>
+<span class="sd"> :type cluster_fields: list of str</span>
+<span class="sd"> """</span>
+
+ <span class="c1"># bigquery only allows certain source formats</span>
+ <span class="c1"># we check to make sure the passed source format is valid</span>
+ <span class="c1"># if it's not, we raise a ValueError</span>
+ <span class="c1"># Refer to this link for more details:</span>
+ <span class="c1"># https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.query.tableDefinitions.(key).sourceFormat</span>
+
+ <span class="k">if</span> <span class="n">schema_fields</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">autodetect</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+ <span class="s1">'You must either pass a schema or autodetect=True.'</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">src_fmt_configs</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">src_fmt_configs</span> <span class="o">=</span> <span class="p">{}</span>
+
+ <span class="n">source_format</span> <span class="o">=</span> <span class="n">source_format</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
+ <span class="n">allowed_formats</span> <span class="o">=</span> <span class="p">[</span>
+ <span class="s2">"CSV"</span><span class="p">,</span> <span class="s2">"NEWLINE_DELIMITED_JSON"</span><span class="p">,</span> <span class="s2">"AVRO"</span><span class="p">,</span> <span class="s2">"GOOGLE_SHEETS"</span><span class="p">,</span>
+ <span class="s2">"DATASTORE_BACKUP"</span><span class="p">,</span> <span class="s2">"PARQUET"</span>
+ <span class="p">]</span>
+ <span class="k">if</span> <span class="n">source_format</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">allowed_formats</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"</span><span class="si">{0}</span><span class="s2"> is not a valid source format. "</span>
+ <span class="s2">"Please use one of the following types: </span><span class="si">{1}</span><span class="s2">"</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">source_format</span><span class="p">,</span> <span class="n">allowed_formats</span><span class="p">))</span>
+
+ <span class="c1"># bigquery also allows you to define how you want a table's schema to change</span>
+ <span class="c1"># as a side effect of a load</span>
+ <span class="c1"># for more details:</span>
+ <span class="c1"># https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load.schemaUpdateOptions</span>
+ <span class="n">allowed_schema_update_options</span> <span class="o">=</span> <span class="p">[</span>
+ <span class="s1">'ALLOW_FIELD_ADDITION'</span><span class="p">,</span> <span class="s2">"ALLOW_FIELD_RELAXATION"</span>
+ <span class="p">]</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="nb">set</span><span class="p">(</span><span class="n">allowed_schema_update_options</span><span class="p">)</span><span class="o">.</span><span class="n">issuperset</span><span class="p">(</span>
+ <span class="nb">set</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">)):</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+ <span class="s2">"</span><span class="si">{0}</span><span class="s2"> contains invalid schema update options."</span>
+ <span class="s2">"Please only use one or more of the following options: </span><span class="si">{1}</span><span class="s2">"</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">,</span> <span class="n">allowed_schema_update_options</span><span class="p">))</span>
+
+ <span class="n">destination_project</span><span class="p">,</span> <span class="n">destination_dataset</span><span class="p">,</span> <span class="n">destination_table</span> <span class="o">=</span> \
+ <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">destination_project_dataset_table</span><span class="p">,</span>
+ <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">var_name</span><span class="o">=</span><span class="s1">'destination_project_dataset_table'</span><span class="p">)</span>
+
+ <span class="n">configuration</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'load'</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s1">'autodetect'</span><span class="p">:</span> <span class="n">autodetect</span><span class="p">,</span>
+ <span class="s1">'createDisposition'</span><span class="p">:</span> <span class="n">create_disposition</span><span class="p">,</span>
+ <span class="s1">'destinationTable'</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s1">'projectId'</span><span class="p">:</span> <span class="n">destination_project</span><span class="p">,</span>
+ <span class="s1">'datasetId'</span><span class="p">:</span> <span class="n">destination_dataset</span><span class="p">,</span>
+ <span class="s1">'tableId'</span><span class="p">:</span> <span class="n">destination_table</span><span class="p">,</span>
+ <span class="p">},</span>
+ <span class="s1">'sourceFormat'</span><span class="p">:</span> <span class="n">source_format</span><span class="p">,</span>
+ <span class="s1">'sourceUris'</span><span class="p">:</span> <span class="n">source_uris</span><span class="p">,</span>
+ <span class="s1">'writeDisposition'</span><span class="p">:</span> <span class="n">write_disposition</span><span class="p">,</span>
+ <span class="s1">'ignoreUnknownValues'</span><span class="p">:</span> <span class="n">ignore_unknown_values</span>
+ <span class="p">}</span>
+ <span class="p">}</span>
+
+ <span class="n">time_partitioning</span> <span class="o">=</span> <span class="n">_cleanse_time_partitioning</span><span class="p">(</span>
+ <span class="n">destination_project_dataset_table</span><span class="p">,</span>
+ <span class="n">time_partitioning</span>
+ <span class="p">)</span>
+ <span class="k">if</span> <span class="n">time_partitioning</span><span class="p">:</span>
+ <span class="n">configuration</span><span class="p">[</span><span class="s1">'load'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">({</span>
+ <span class="s1">'timePartitioning'</span><span class="p">:</span> <span class="n">time_partitioning</span>
+ <span class="p">})</span>
+
+ <span class="k">if</span> <span class="n">cluster_fields</span><span class="p">:</span>
+ <span class="n">configuration</span><span class="p">[</span><span class="s1">'load'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s1">'clustering'</span><span class="p">:</span> <span class="p">{</span><span class="s1">'fields'</span><span class="p">:</span> <span class="n">cluster_fields</span><span class="p">}})</span>
+
+ <span class="k">if</span> <span class="n">schema_fields</span><span class="p">:</span>
+ <span class="n">configuration</span><span class="p">[</span><span class="s1">'load'</span><span class="p">][</span><span class="s1">'schema'</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'fields'</span><span class="p">:</span> <span class="n">schema_fields</span><span class="p">}</span>
+
+ <span class="k">if</span> <span class="n">schema_update_options</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">write_disposition</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">"WRITE_APPEND"</span><span class="p">,</span> <span class="s2">"WRITE_TRUNCATE"</span><span class="p">]:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"schema_update_options is only "</span>
+ <span class="s2">"allowed if write_disposition is "</span>
+ <span class="s2">"'WRITE_APPEND' or 'WRITE_TRUNCATE'."</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s2">"Adding experimental "</span>
+ <span class="s2">"'schemaUpdateOptions': </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">))</span>
+ <span class="n">configuration</span><span class="p">[</span><span class="s1">'load'</span><span class="p">][</span>
+ <span class="s1">'schemaUpdateOptions'</span><span class="p">]</span> <span class="o">=</span> <span class="n">schema_update_options</span>
+
+ <span class="k">if</span> <span class="n">max_bad_records</span><span class="p">:</span>
+ <span class="n">configuration</span><span class="p">[</span><span class="s1">'load'</span><span class="p">][</span><span class="s1">'maxBadRecords'</span><span class="p">]</span> <span class="o">=</span> <span class="n">max_bad_records</span>
+
+ <span class="c1"># if following fields are not specified in src_fmt_configs,</span>
+ <span class="c1"># honor the top-level params for backward-compatibility</span>
+ <span class="k">if</span> <span class="s1">'skipLeadingRows'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
+ <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">'skipLeadingRows'</span><span class="p">]</span> <span class="o">=</span> <span class="n">skip_leading_rows</span>
+ <span class="k">if</span> <span class="s1">'fieldDelimiter'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
+ <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">'fieldDelimiter'</span><span class="p">]</span> <span class="o">=</span> <span class="n">field_delimiter</span>
+ <span class="k">if</span> <span class="s1">'ignoreUnknownValues'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
+ <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">'ignoreUnknownValues'</span><span class="p">]</span> <span class="o">=</span> <span class="n">ignore_unknown_values</span>
+ <span class="k">if</span> <span class="n">quote_character</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">'quote'</span><span class="p">]</span> <span class="o">=</span> <span class="n">quote_character</span>
+ <span class="k">if</span> <span class="n">allow_quoted_newlines</span><span class="p">:</span>
+ <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">'allowQuotedNewlines'</span><span class="p">]</span> <span class="o">=</span> <span class="n">allow_quoted_newlines</span>
+
+ <span class="n">src_fmt_to_configs_mapping</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'CSV'</span><span class="p">:</span> <span class="p">[</span>
+ <span class="s1">'allowJaggedRows'</span><span class="p">,</span> <span class="s1">'allowQuotedNewlines'</span><span class="p">,</span> <span class="s1">'autodetect'</span><span class="p">,</span>
+ <span class="s1">'fieldDelimiter'</span><span class="p">,</span> <span class="s1">'skipLeadingRows'</span><span class="p">,</span> <span class="s1">'ignoreUnknownValues'</span><span class="p">,</span>
+ <span class="s1">'nullMarker'</span><span class="p">,</span> <span class="s1">'quote'</span>
+ <span class="p">],</span>
+ <span class="s1">'DATASTORE_BACKUP'</span><span class="p">:</span> <span class="p">[</span><span class="s1">'projectionFields'</span><span class="p">],</span>
+ <span class="s1">'NEWLINE_DELIMITED_JSON'</span><span class="p">:</span> <span class="p">[</span><span class="s1">'autodetect'</span><span class="p">,</span> <span class="s1">'ignoreUnknownValues'</span><span class="p">],</span>
+ <span class="s1">'PARQUET'</span><span class="p">:</span> <span class="p">[</span><span class="s1">'autodetect'</span><span class="p">,</span> <span class="s1">'ignoreUnknownValues'</span><span class="p">],</span>
+ <span class="s1">'AVRO'</span><span class="p">:</span> <span class="p">[],</span>
+ <span class="p">}</span>
+ <span class="n">valid_configs</span> <span class="o">=</span> <span class="n">src_fmt_to_configs_mapping</span><span class="p">[</span><span class="n">source_format</span><span class="p">]</span>
+ <span class="n">src_fmt_configs</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="n">k</span><span class="p">:</span> <span class="n">v</span>
+ <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="k">if</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">valid_configs</span>
+ <span class="p">}</span>
+ <span class="n">configuration</span><span class="p">[</span><span class="s1">'load'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">src_fmt_configs</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">allow_jagged_rows</span><span class="p">:</span>
+ <span class="n">configuration</span><span class="p">[</span><span class="s1">'load'</span><span class="p">][</span><span class="s1">'allowJaggedRows'</span><span class="p">]</span> <span class="o">=</span> <span class="n">allow_jagged_rows</span>
+
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_with_configuration</span><span class="p">(</span><span class="n">configuration</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">run_with_configuration</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">configuration</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Executes a BigQuery SQL query. See here:</span>
+
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/v2/jobs</span>
+
+<span class="sd"> For more details about the configuration parameter.</span>
+
+<span class="sd"> :param configuration: The configuration parameter maps directly to</span>
+<span class="sd"> BigQuery's configuration field in the job object. See</span>
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/v2/jobs for</span>
+<span class="sd"> details.</span>
+<span class="sd"> """</span>
+ <span class="n">jobs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span>
+ <span class="n">job_data</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'configuration'</span><span class="p">:</span> <span class="n">configuration</span><span class="p">}</span>
+
+ <span class="c1"># Send query and wait for reply.</span>
+ <span class="n">query_reply</span> <span class="o">=</span> <span class="n">jobs</span> \
+ <span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">job_data</span><span class="p">)</span> \
+ <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span> <span class="o">=</span> <span class="n">query_reply</span><span class="p">[</span><span class="s1">'jobReference'</span><span class="p">][</span><span class="s1">'jobId'</span><span class="p">]</span>
+
+ <span class="c1"># Wait for query to finish.</span>
+ <span class="n">keep_polling_job</span> <span class="o">=</span> <span class="kc">True</span>
+ <span class="k">while</span> <span class="n">keep_polling_job</span><span class="p">:</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">:</span>
+ <span class="n">job</span> <span class="o">=</span> <span class="n">jobs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">jobId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span><span class="p">,</span>
+ <span class="n">location</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">job</span> <span class="o">=</span> <span class="n">jobs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">jobId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">if</span> <span class="n">job</span><span class="p">[</span><span class="s1">'status'</span><span class="p">][</span><span class="s1">'state'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'DONE'</span><span class="p">:</span>
+ <span class="n">keep_polling_job</span> <span class="o">=</span> <span class="kc">False</span>
+ <span class="c1"># Check if job had errors.</span>
+ <span class="k">if</span> <span class="s1">'errorResult'</span> <span class="ow">in</span> <span class="n">job</span><span class="p">[</span><span class="s1">'status'</span><span class="p">]:</span>
+ <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
+ <span class="s1">'BigQuery job failed. Final error was: </span><span class="si">{}</span><span class="s1">. The job was: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span>
+ <span class="nb">format</span><span class="p">(</span><span class="n">job</span><span class="p">[</span><span class="s1">'status'</span><span class="p">][</span><span class="s1">'errorResult'</span><span class="p">],</span> <span class="n">job</span><span class="p">))</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Waiting for job to complete : </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span><span class="p">)</span>
+ <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
+
+ <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">err</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span> <span class="ow">in</span> <span class="p">[</span><span class="mi">500</span><span class="p">,</span> <span class="mi">503</span><span class="p">]:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'</span><span class="si">%s</span><span class="s1">: Retryable error, waiting for job to complete: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">err</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span><span class="p">)</span>
+ <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
+ <span class="s1">'BigQuery job status check failed. Final error was: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">err</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span>
+
+ <span class="k">def</span> <span class="nf">poll_job_complete</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">job_id</span><span class="p">):</span>
+ <span class="n">jobs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">:</span>
+ <span class="n">job</span> <span class="o">=</span> <span class="n">jobs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">jobId</span><span class="o">=</span><span class="n">job_id</span><span class="p">,</span>
+ <span class="n">location</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">job</span> <span class="o">=</span> <span class="n">jobs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">jobId</span><span class="o">=</span><span class="n">job_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">if</span> <span class="n">job</span><span class="p">[</span><span class="s1">'status'</span><span class="p">][</span><span class="s1">'state'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'DONE'</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">True</span>
+ <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">err</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span> <span class="ow">in</span> <span class="p">[</span><span class="mi">500</span><span class="p">,</span> <span class="mi">503</span><span class="p">]:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'</span><span class="si">%s</span><span class="s1">: Retryable error while polling job with id </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">err</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span><span class="p">,</span> <span class="n">job_id</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
+ <span class="s1">'BigQuery job status check failed. Final error was: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">err</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span><span class="p">)</span>
+ <span class="k">return</span> <span class="kc">False</span>
+
+ <span class="k">def</span> <span class="nf">cancel_query</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Cancel all started queries that have not yet completed</span>
+<span class="sd"> """</span>
+ <span class="n">jobs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span>
+ <span class="k">if</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span> <span class="ow">and</span>
+ <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">poll_job_complete</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span><span class="p">)):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Attempting to cancel job : </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">:</span>
+ <span class="n">jobs</span><span class="o">.</span><span class="n">cancel</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">jobId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span><span class="p">,</span>
+ <span class="n">location</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">jobs</span><span class="o">.</span><span class="n">cancel</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">jobId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'No running BigQuery jobs to cancel.'</span><span class="p">)</span>
+ <span class="k">return</span>
+
+ <span class="c1"># Wait for all the calls to cancel to finish</span>
+ <span class="n">max_polling_attempts</span> <span class="o">=</span> <span class="mi">12</span>
+ <span class="n">polling_attempts</span> <span class="o">=</span> <span class="mi">0</span>
+
+ <span class="n">job_complete</span> <span class="o">=</span> <span class="kc">False</span>
+ <span class="k">while</span> <span class="n">polling_attempts</span> <span class="o"><</span> <span class="n">max_polling_attempts</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">job_complete</span><span class="p">:</span>
+ <span class="n">polling_attempts</span> <span class="o">=</span> <span class="n">polling_attempts</span> <span class="o">+</span> <span class="mi">1</span>
+ <span class="n">job_complete</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">poll_job_complete</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">job_complete</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Job successfully canceled: </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="n">polling_attempts</span> <span class="o">==</span> <span class="n">max_polling_attempts</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s2">"Stopping polling due to timeout. Job with id </span><span class="si">%s</span><span class="s2"> "</span>
+ <span class="s2">"has not completed cancel and may or may not finish."</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Waiting for canceled job with id </span><span class="si">%s</span><span class="s1"> to finish.'</span><span class="p">,</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span><span class="p">)</span>
+ <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">get_schema</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Get the schema for a given datset.table.</span>
+<span class="sd"> see https://cloud.google.com/bigquery/docs/reference/v2/tables#resource</span>
+
+<span class="sd"> :param dataset_id: the dataset ID of the requested table</span>
+<span class="sd"> :param table_id: the table ID of the requested table</span>
+<span class="sd"> :return: a table schema</span>
+<span class="sd"> """</span>
+ <span class="n">tables_resource</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span> \
+ <span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span> <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span><span class="p">)</span> \
+ <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">tables_resource</span><span class="p">[</span><span class="s1">'schema'</span><span class="p">]</span>
+
+ <span class="k">def</span> <span class="nf">get_tabledata</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">,</span>
+ <span class="n">max_results</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">selected_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">page_token</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">start_index</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Get the data of a given dataset.table and optionally with selected columns.</span>
+<span class="sd"> see https://cloud.google.com/bigquery/docs/reference/v2/tabledata/list</span>
+
+<span class="sd"> :param dataset_id: the dataset ID of the requested table.</span>
+<span class="sd"> :param table_id: the table ID of the requested table.</span>
+<span class="sd"> :param max_results: the maximum results to return.</span>
+<span class="sd"> :param selected_fields: List of fields to return (comma-separated). If</span>
+<span class="sd"> unspecified, all fields are returned.</span>
+<span class="sd"> :param page_token: page token, returned from a previous call,</span>
+<span class="sd"> identifying the result set.</span>
+<span class="sd"> :param start_index: zero based index of the starting row to read.</span>
+<span class="sd"> :return: map containing the requested rows.</span>
+<span class="sd"> """</span>
+ <span class="n">optional_params</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="k">if</span> <span class="n">max_results</span><span class="p">:</span>
+ <span class="n">optional_params</span><span class="p">[</span><span class="s1">'maxResults'</span><span class="p">]</span> <span class="o">=</span> <span class="n">max_results</span>
+ <span class="k">if</span> <span class="n">selected_fields</span><span class="p">:</span>
+ <span class="n">optional_params</span><span class="p">[</span><span class="s1">'selectedFields'</span><span class="p">]</span> <span class="o">=</span> <span class="n">selected_fields</span>
+ <span class="k">if</span> <span class="n">page_token</span><span class="p">:</span>
+ <span class="n">optional_params</span><span class="p">[</span><span class="s1">'pageToken'</span><span class="p">]</span> <span class="o">=</span> <span class="n">page_token</span>
+ <span class="k">if</span> <span class="n">start_index</span><span class="p">:</span>
+ <span class="n">optional_params</span><span class="p">[</span><span class="s1">'startIndex'</span><span class="p">]</span> <span class="o">=</span> <span class="n">start_index</span>
+ <span class="k">return</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tabledata</span><span class="p">()</span><span class="o">.</span><span class="n">list</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+ <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">optional_params</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">())</span>
+
+ <span class="k">def</span> <span class="nf">run_table_delete</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">deletion_dataset_table</span><span class="p">,</span>
+ <span class="n">ignore_if_missing</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Delete an existing table from the dataset;</span>
+<span class="sd"> If the table does not exist, return an error unless ignore_if_missing</span>
+<span class="sd"> is set to True.</span>
+
+<span class="sd"> :param deletion_dataset_table: A dotted</span>
+<span class="sd"> (<project>.|<project>:)<dataset>.<table> that indicates which table</span>
+<span class="sd"> will be deleted.</span>
+<span class="sd"> :type deletion_dataset_table: str</span>
+<span class="sd"> :param ignore_if_missing: if True, then return success even if the</span>
+<span class="sd"> requested table does not exist.</span>
+<span class="sd"> :type ignore_if_missing: boolean</span>
+<span class="sd"> :return:</span>
+<span class="sd"> """</span>
+ <span class="n">deletion_project</span><span class="p">,</span> <span class="n">deletion_dataset</span><span class="p">,</span> <span class="n">deletion_table</span> <span class="o">=</span> \
+ <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">deletion_dataset_table</span><span class="p">,</span>
+ <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">)</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span> \
+ <span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">deletion_project</span><span class="p">,</span>
+ <span class="n">datasetId</span><span class="o">=</span><span class="n">deletion_dataset</span><span class="p">,</span>
+ <span class="n">tableId</span><span class="o">=</span><span class="n">deletion_table</span><span class="p">)</span> \
+ <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Deleted table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1">.'</span><span class="p">,</span> <span class="n">deletion_project</span><span class="p">,</span>
+ <span class="n">deletion_dataset</span><span class="p">,</span> <span class="n">deletion_table</span><span class="p">)</span>
+ <span class="k">except</span> <span class="n">HttpError</span><span class="p">:</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">ignore_if_missing</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s1">'Table deletion failed. Table does not exist.'</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Table does not exist. Skipping.'</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">run_table_upsert</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_resource</span><span class="p">,</span> <span class="n">project_id</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> creates a new, empty table in the dataset;</span>
+<span class="sd"> If the table already exists, update the existing table.</span>
+<span class="sd"> Since BigQuery does not natively allow table upserts, this is not an</span>
+<span class="sd"> atomic operation.</span>
+
+<span class="sd"> :param dataset_id: the dataset to upsert the table into.</span>
+<span class="sd"> :type dataset_id: str</span>
+<span class="sd"> :param table_resource: a table resource. see</span>
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/v2/tables#resource</span>
+<span class="sd"> :type table_resource: dict</span>
+<span class="sd"> :param project_id: the project to upsert the table into. If None,</span>
+<span class="sd"> project will be self.project_id.</span>
+<span class="sd"> :return:</span>
+<span class="sd"> """</span>
+ <span class="c1"># check to see if the table exists</span>
+ <span class="n">table_id</span> <span class="o">=</span> <span class="n">table_resource</span><span class="p">[</span><span class="s1">'tableReference'</span><span class="p">][</span><span class="s1">'tableId'</span><span class="p">]</span>
+ <span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span> <span class="k">if</span> <span class="n">project_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
+ <span class="n">tables_list_resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">list</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span> <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+ <span class="k">for</span> <span class="n">table</span> <span class="ow">in</span> <span class="n">tables_list_resp</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'tables'</span><span class="p">,</span> <span class="p">[]):</span>
+ <span class="k">if</span> <span class="n">table</span><span class="p">[</span><span class="s1">'tableReference'</span><span class="p">][</span><span class="s1">'tableId'</span><span class="p">]</span> <span class="o">==</span> <span class="n">table_id</span><span class="p">:</span>
+ <span class="c1"># found the table, do update</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> exists, updating.'</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+ <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span><span class="p">,</span>
+ <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="c1"># If there is a next page, we need to check the next page.</span>
+ <span class="k">if</span> <span class="s1">'nextPageToken'</span> <span class="ow">in</span> <span class="n">tables_list_resp</span><span class="p">:</span>
+ <span class="n">tables_list_resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span>\
+ <span class="o">.</span><span class="n">list</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+ <span class="n">pageToken</span><span class="o">=</span><span class="n">tables_list_resp</span><span class="p">[</span><span class="s1">'nextPageToken'</span><span class="p">])</span>\
+ <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="c1"># If there is no next page, then the table doesn't exist.</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="c1"># do insert</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> does not exist. creating.'</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+ <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+
+ <span class="k">def</span> <span class="nf">run_grant_dataset_view_access</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">source_dataset</span><span class="p">,</span>
+ <span class="n">view_dataset</span><span class="p">,</span>
+ <span class="n">view_table</span><span class="p">,</span>
+ <span class="n">source_project</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">view_project</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Grant authorized view access of a dataset to a view table.</span>
+<span class="sd"> If this view has already been granted access to the dataset, do nothing.</span>
+<span class="sd"> This method is not atomic. Running it may clobber a simultaneous update.</span>
+
+<span class="sd"> :param source_dataset: the source dataset</span>
+<span class="sd"> :type source_dataset: str</span>
+<span class="sd"> :param view_dataset: the dataset that the view is in</span>
+<span class="sd"> :type view_dataset: str</span>
+<span class="sd"> :param view_table: the table of the view</span>
+<span class="sd"> :type view_table: str</span>
+<span class="sd"> :param source_project: the project of the source dataset. If None,</span>
+<span class="sd"> self.project_id will be used.</span>
+<span class="sd"> :type source_project: str</span>
+<span class="sd"> :param view_project: the project that the view is in. If None,</span>
+<span class="sd"> self.project_id will be used.</span>
+<span class="sd"> :type view_project: str</span>
+<span class="sd"> :return: the datasets resource of the source dataset.</span>
+<span class="sd"> """</span>
+
+ <span class="c1"># Apply default values to projects</span>
+ <span class="n">source_project</span> <span class="o">=</span> <span class="n">source_project</span> <span class="k">if</span> <span class="n">source_project</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
+ <span class="n">view_project</span> <span class="o">=</span> <span class="n">view_project</span> <span class="k">if</span> <span class="n">view_project</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
+
+ <span class="c1"># we don't want to clobber any existing accesses, so we have to get</span>
+ <span class="c1"># info on the dataset before we can add view access</span>
+ <span class="n">source_dataset_resource</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">datasets</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="n">source_project</span><span class="p">,</span> <span class="n">datasetId</span><span class="o">=</span><span class="n">source_dataset</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="n">access</span> <span class="o">=</span> <span class="n">source_dataset_resource</span><span class="p">[</span>
+ <span class="s1">'access'</span><span class="p">]</span> <span class="k">if</span> <span class="s1">'access'</span> <span class="ow">in</span> <span class="n">source_dataset_resource</span> <span class="k">else</span> <span class="p">[]</span>
+ <span class="n">view_access</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'view'</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s1">'projectId'</span><span class="p">:</span> <span class="n">view_project</span><span class="p">,</span>
+ <span class="s1">'datasetId'</span><span class="p">:</span> <span class="n">view_dataset</span><span class="p">,</span>
+ <span class="s1">'tableId'</span><span class="p">:</span> <span class="n">view_table</span>
+ <span class="p">}</span>
+ <span class="p">}</span>
+ <span class="c1"># check to see if the view we want to add already exists.</span>
+ <span class="k">if</span> <span class="n">view_access</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">access</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'Granting table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> authorized view access to </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1"> dataset.'</span><span class="p">,</span>
+ <span class="n">view_project</span><span class="p">,</span> <span class="n">view_dataset</span><span class="p">,</span> <span class="n">view_table</span><span class="p">,</span> <span class="n">source_project</span><span class="p">,</span>
+ <span class="n">source_dataset</span><span class="p">)</span>
+ <span class="n">access</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">view_access</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">datasets</span><span class="p">()</span><span class="o">.</span><span class="n">patch</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="n">source_project</span><span class="p">,</span>
+ <span class="n">datasetId</span><span class="o">=</span><span class="n">source_dataset</span><span class="p">,</span>
+ <span class="n">body</span><span class="o">=</span><span class="p">{</span>
+ <span class="s1">'access'</span><span class="p">:</span> <span class="n">access</span>
+ <span class="p">})</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="c1"># if view is already in access, do nothing.</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s1">'Table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> already has authorized view access to </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1"> dataset.'</span><span class="p">,</span>
+ <span class="n">view_project</span><span class="p">,</span> <span class="n">view_dataset</span><span class="p">,</span> <span class="n">view_table</span><span class="p">,</span> <span class="n">source_project</span><span class="p">,</span> <span class="n">source_dataset</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">source_dataset_resource</span>
+
+ <span class="k">def</span> <span class="nf">create_empty_dataset</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_id</span><span class="o">=</span><span class="s2">""</span><span class="p">,</span> <span class="n">project_id</span><span class="o">=</span><span class="s2">""</span><span class="p">,</span>
+ <span class="n">dataset_reference</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Create a new empty dataset:</span>
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets/insert</span>
+
+<span class="sd"> :param project_id: The name of the project where we want to create</span>
+<span class="sd"> an empty a dataset. Don't need to provide, if projectId in dataset_reference.</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :param dataset_id: The id of dataset. Don't need to provide,</span>
+<span class="sd"> if datasetId in dataset_reference.</span>
+<span class="sd"> :type dataset_id: str</span>
+<span class="sd"> :param dataset_reference: Dataset reference that could be provided</span>
+<span class="sd"> with request body. More info:</span>
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets#resource</span>
+<span class="sd"> :type dataset_reference: dict</span>
+<span class="sd"> """</span>
+
+ <span class="k">if</span> <span class="n">dataset_reference</span><span class="p">:</span>
+ <span class="n">_validate_value</span><span class="p">(</span><span class="s1">'dataset_reference'</span><span class="p">,</span> <span class="n">dataset_reference</span><span class="p">,</span> <span class="nb">dict</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">dataset_reference</span> <span class="o">=</span> <span class="p">{}</span>
+
+ <span class="k">if</span> <span class="s2">"datasetReference"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">dataset_reference</span><span class="p">:</span>
+ <span class="n">dataset_reference</span><span class="p">[</span><span class="s2">"datasetReference"</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">dataset_reference</span><span class="p">[</span><span class="s2">"datasetReference"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"datasetId"</span><span class="p">)</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">dataset_id</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+ <span class="s2">"</span><span class="si">{}</span><span class="s2"> not provided datasetId. Impossible to create dataset"</span><span class="p">)</span>
+
+ <span class="n">dataset_required_params</span> <span class="o">=</span> <span class="p">[(</span><span class="n">dataset_id</span><span class="p">,</span> <span class="s2">"datasetId"</span><span class="p">,</span> <span class="s2">""</span><span class="p">),</span>
+ <span class="p">(</span><span class="n">project_id</span><span class="p">,</span> <span class="s2">"projectId"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">)]</span>
+ <span class="k">for</span> <span class="n">param_tuple</span> <span class="ow">in</span> <span class="n">dataset_required_params</span><span class="p">:</span>
+ <span class="n">param</span><span class="p">,</span> <span class="n">param_name</span><span class="p">,</span> <span class="n">param_default</span> <span class="o">=</span> <span class="n">param_tuple</span>
+ <span class="k">if</span> <span class="n">param_name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">dataset_reference</span><span class="p">[</span><span class="s1">'datasetReference'</span><span class="p">]:</span>
+ <span class="k">if</span> <span class="n">param_default</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">param</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"</span><span class="si">{}</span><span class="s2"> was not specified. Will be used default "</span>
+ <span class="s2">"value </span><span class="si">{}</span><span class="s2">."</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">param_name</span><span class="p">,</span>
+ <span class="n">param_default</span><span class="p">))</span>
+ <span class="n">param</span> <span class="o">=</span> <span class="n">param_default</span>
+ <span class="n">dataset_reference</span><span class="p">[</span><span class="s1">'datasetReference'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
+ <span class="p">{</span><span class="n">param_name</span><span class="p">:</span> <span class="n">param</span><span class="p">})</span>
+ <span class="k">elif</span> <span class="n">param</span><span class="p">:</span>
+ <span class="n">_api_resource_configs_duplication_check</span><span class="p">(</span>
+ <span class="n">param_name</span><span class="p">,</span> <span class="n">param</span><span class="p">,</span>
+ <span class="n">dataset_reference</span><span class="p">[</span><span class="s1">'datasetReference'</span><span class="p">],</span> <span class="s1">'dataset_reference'</span><span class="p">)</span>
+
+ <span class="n">dataset_id</span> <span class="o">=</span> <span class="n">dataset_reference</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"datasetReference"</span><span class="p">)</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"datasetId"</span><span class="p">)</span>
+ <span class="n">dataset_project_id</span> <span class="o">=</span> <span class="n">dataset_reference</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"datasetReference"</span><span class="p">)</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="s2">"projectId"</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Creating Dataset: </span><span class="si">%s</span><span class="s1"> in project: </span><span class="si">%s</span><span class="s1"> '</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span>
+ <span class="n">dataset_project_id</span><span class="p">)</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">datasets</span><span class="p">()</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="n">dataset_project_id</span><span class="p">,</span>
+ <span class="n">body</span><span class="o">=</span><span class="n">dataset_reference</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Dataset created successfully: In project </span><span class="si">%s</span><span class="s1"> '</span>
+ <span class="s1">'Dataset </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">dataset_project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">)</span>
+
+ <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'BigQuery job failed. Error was: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">err</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
+ <span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">delete_dataset</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Delete a dataset of Big query in your project.</span>
+<span class="sd"> :param project_id: The name of the project where we have the dataset .</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :param dataset_id: The dataset to be delete.</span>
+<span class="sd"> :type dataset_id: str</span>
+<span class="sd"> :return:</span>
+<span class="sd"> """</span>
+ <span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span> <span class="k">if</span> <span class="n">project_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Deleting from project: </span><span class="si">%s</span><span class="s1"> Dataset:</span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">)</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">datasets</span><span class="p">()</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Dataset deleted successfully: In project </span><span class="si">%s</span><span class="s1"> '</span>
+ <span class="s1">'Dataset </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">)</span>
+
+ <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'BigQuery job failed. Error was: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">err</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
+ <span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">get_dataset</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">project_id</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Method returns dataset_resource if dataset exist</span>
+<span class="sd"> and raised 404 error if dataset does not exist</span>
+
+<span class="sd"> :param dataset_id: The BigQuery Dataset ID</span>
+<span class="sd"> :type dataset_id: str</span>
+<span class="sd"> :param project_id: The GCP Project ID</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :return: dataset_resource</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> For more information, see Dataset Resource content:</span>
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets#resource</span>
+<span class="sd"> """</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">dataset_id</span> <span class="ow">or</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dataset_id</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"dataset_id argument must be provided and has "</span>
+ <span class="s2">"a type 'str'. You provided: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">dataset_id</span><span class="p">))</span>
+
+ <span class="n">dataset_project_id</span> <span class="o">=</span> <span class="n">project_id</span> <span class="k">if</span> <span class="n">project_id</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">dataset_resource</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">datasets</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span> <span class="n">projectId</span><span class="o">=</span><span class="n">dataset_project_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Dataset Resource: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">dataset_resource</span><span class="p">))</span>
+ <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'BigQuery job failed. Error was: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">err</span><span class="o">.</span><span class="n">content</span><span class="p">))</span>
+
+ <span class="k">return</span> <span class="n">dataset_resource</span>
+
+ <span class="k">def</span> <span class="nf">get_datasets_list</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Method returns full list of BigQuery datasets in the current project</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> For more information, see:</span>
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets/list</span>
+
+<span class="sd"> :param project_id: Google Cloud Project for which you</span>
+<span class="sd"> try to get all datasets</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :return: datasets_list</span>
+
+<span class="sd"> Example of returned datasets_list: ::</span>
+
+<span class="sd"> {</span>
+<span class="sd"> "kind":"bigquery#dataset",</span>
+<span class="sd"> "location":"US",</span>
+<span class="sd"> "id":"your-project:dataset_2_test",</span>
+<span class="sd"> "datasetReference":{</span>
+<span class="sd"> "projectId":"your-project",</span>
+<span class="sd"> "datasetId":"dataset_2_test"</span>
+<span class="sd"> }</span>
+<span class="sd"> },</span>
+<span class="sd"> {</span>
+<span class="sd"> "kind":"bigquery#dataset",</span>
+<span class="sd"> "location":"US",</span>
+<span class="sd"> "id":"your-project:dataset_1_test",</span>
+<span class="sd"> "datasetReference":{</span>
+<span class="sd"> "projectId":"your-project",</span>
+<span class="sd"> "datasetId":"dataset_1_test"</span>
+<span class="sd"> }</span>
+<span class="sd"> }</span>
+<span class="sd"> ]</span>
+<span class="sd"> """</span>
+ <span class="n">dataset_project_id</span> <span class="o">=</span> <span class="n">project_id</span> <span class="k">if</span> <span class="n">project_id</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">datasets_list</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">datasets</span><span class="p">()</span><span class="o">.</span><span class="n">list</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="n">dataset_project_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()[</span><span class="s1">'datasets'</span><span class="p">]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Datasets List: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">datasets_list</span><span class="p">))</span>
+
+ <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'BigQuery job failed. Error was: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">err</span><span class="o">.</span><span class="n">content</span><span class="p">))</span>
+
+ <span class="k">return</span> <span class="n">datasets_list</span>
+
+ <span class="k">def</span> <span class="nf">insert_all</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">,</span>
+ <span class="n">rows</span><span class="p">,</span> <span class="n">ignore_unknown_values</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">skip_invalid_rows</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">fail_on_error</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Method to stream data into BigQuery one record at a time without needing</span>
+<span class="sd"> to run a load job</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> For more information, see:</span>
+<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll</span>
+
+<span class="sd"> :param project_id: The name of the project where we have the table</span>
+<span class="sd"> :type project_id: str</span>
+<span class="sd"> :param dataset_id: The name of the dataset where we have the table</span>
+<span class="sd"> :type dataset_id: str</span>
+<span class="sd"> :param table_id: The name of the table</span>
+<span class="sd"> :type table_id: str</span>
+<span class="sd"> :param rows: the rows to insert</span>
+<span class="sd"> :type rows: list</span>
+
+<span class="sd"> **Example or rows**:</span>
+<span class="sd"> rows=[{"json": {"a_key": "a_value_0"}}, {"json": {"a_key": "a_value_1"}}]</span>
+
+<span class="sd"> :param ignore_unknown_values: [Optional] Accept rows that contain values</span>
+<span class="sd"> that do not match the schema. The unknown values are ignored.</span>
+<span class="sd"> The default value is false, which treats unknown values as errors.</span>
+<span class="sd"> :type ignore_unknown_values: bool</span>
+<span class="sd"> :param skip_invalid_rows: [Optional] Insert all valid rows of a request,</span>
+<span class="sd"> even if invalid rows exist. The default value is false, which causes</span>
+<span class="sd"> the entire request to fail if any invalid rows exist.</span>
+<span class="sd"> :type skip_invalid_rows: bool</span>
+<span class="sd"> :param fail_on_error: [Optional] Force the task to fail if any errors occur.</span>
+<span class="sd"> The default value is false, which indicates the task should not fail</span>
+<span class="sd"> even if any insertion errors occur.</span>
+<span class="sd"> :type fail_on_error: bool</span>
+<span class="sd"> """</span>
+
+ <span class="n">dataset_project_id</span> <span class="o">=</span> <span class="n">project_id</span> <span class="k">if</span> <span class="n">project_id</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
+
+ <span class="n">body</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s2">"rows"</span><span class="p">:</span> <span class="n">rows</span><span class="p">,</span>
+ <span class="s2">"ignoreUnknownValues"</span><span class="p">:</span> <span class="n">ignore_unknown_values</span><span class="p">,</span>
+ <span class="s2">"kind"</span><span class="p">:</span> <span class="s2">"bigquery#tableDataInsertAllRequest"</span><span class="p">,</span>
+ <span class="s2">"skipInvalidRows"</span><span class="p">:</span> <span class="n">skip_invalid_rows</span><span class="p">,</span>
+ <span class="p">}</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Inserting </span><span class="si">{}</span><span class="s1"> row(s) into Table </span><span class="si">{}</span><span class="s1">:</span><span class="si">{}</span><span class="s1">.</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span cla [...]
+ <span class="nb">len</span><span class="p">(</span><span class="n">rows</span><span class="p">),</span> <span class="n">dataset_project_id</span><span class="p">,</span>
+ <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">))</span>
+
+ <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tabledata</span><span class="p">()</span><span class="o">.</span><span class="n">insertAll</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="n">dataset_project_id</span><span class="p">,</span> <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+ <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+
+ <span class="k">if</span> <span class="s1">'insertErrors'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">resp</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'All row(s) inserted successfully: </span><span class="si">{}</span><span class="s1">:</span><span class="si">{}</span><span class="s1">.</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">dataset_project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">))</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">error_msg</span> <span class="o">=</span> <span class="s1">'</span><span class="si">{}</span><span class="s1"> insert error(s) occured: </span><span class="si">{}</span><span class="s1">:</span><span class="si">{}</span><span class="s1">.</span><span class="si">{}</span><span class="s1">. Details: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="nb">len</span><span class="p">(</span><span class="n">resp</span><span class="p">[</span><span class="s1">'insertErrors'</span><span class="p">]),</span>
+ <span class="n">dataset_project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">,</span> <span class="n">resp</span><span class="p">[</span><span class="s1">'insertErrors'</span><span class="p">])</span>
+ <span class="k">if</span> <span class="n">fail_on_error</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'BigQuery job failed. Error was: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_msg</span><span class="p">)</span>
+ <span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">error_msg</span><span class="p">)</span>
+ <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'BigQuery job failed. Error was: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">err</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
+ <span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">BigQueryCursor</span><span class="p">(</span><span class="n">BigQueryBaseCursor</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> A very basic BigQuery PEP 249 cursor implementation. The PyHive PEP 249</span>
+<span class="sd"> implementation was used as a reference:</span>
+
+<span class="sd"> https://github.com/dropbox/PyHive/blob/master/pyhive/presto.py</span>
+<span class="sd"> https://github.com/dropbox/PyHive/blob/master/pyhive/common.py</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">service</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">use_legacy_sql</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">location</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryCursor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+ <span class="n">service</span><span class="o">=</span><span class="n">service</span><span class="p">,</span>
+ <span class="n">project_id</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">use_legacy_sql</span><span class="o">=</span><span class="n">use_legacy_sql</span><span class="p">,</span>
+ <span class="n">location</span><span class="o">=</span><span class="n">location</span><span class="p">,</span>
+ <span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">buffersize</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">page_token</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">buffer</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">all_pages_loaded</span> <span class="o">=</span> <span class="kc">False</span>
+
+ <span class="nd">@property</span>
+ <span class="k">def</span> <span class="nf">description</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">""" The schema description method is not currently implemented. """</span>
+ <span class="k">raise</span> <span class="ne">NotImplementedError</span>
+
+ <span class="k">def</span> <span class="nf">close</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">""" By default, do nothing """</span>
+ <span class="k">pass</span>
+
+ <span class="nd">@property</span>
+ <span class="k">def</span> <span class="nf">rowcount</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">""" By default, return -1 to indicate that this is not supported. """</span>
+ <span class="k">return</span> <span class="o">-</span><span class="mi">1</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">operation</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Executes a BigQuery query, and returns the job ID.</span>
+
+<span class="sd"> :param operation: The query to execute.</span>
+<span class="sd"> :type operation: string</span>
+<span class="sd"> :param parameters: Parameters to substitute into the query.</span>
+<span class="sd"> :type parameters: dict</span>
+<span class="sd"> """</span>
+ <span class="n">sql</span> <span class="o">=</span> <span class="n">_bind_parameters</span><span class="p">(</span><span class="n">operation</span><span class="p">,</span>
+ <span class="n">parameters</span><span class="p">)</span> <span class="k">if</span> <span class="n">parameters</span> <span class="k">else</span> <span class="n">operation</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_query</span><span class="p">(</span><span class="n">sql</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">executemany</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">operation</span><span class="p">,</span> <span class="n">seq_of_parameters</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Execute a BigQuery query multiple times with different parameters.</span>
+
+<span class="sd"> :param operation: The query to execute.</span>
+<span class="sd"> :type operation: string</span>
+<span class="sd"> :param seq_of_parameters: List of dictionary parameters to substitute into the</span>
+<span class="sd"> query.</span>
+<span class="sd"> :type seq_of_parameters: list</span>
+<span class="sd"> """</span>
+ <span class="k">for</span> <span class="n">parameters</span> <span class="ow">in</span> <span class="n">seq_of_parameters</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">operation</span><span class="p">,</span> <span class="n">parameters</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">fetchone</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">""" Fetch the next row of a query result set. """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">next</span><span class="p">()</span>
+
+ <span class="k">def</span> <span class="nf">next</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Helper method for fetchone, which returns the next row from a buffer.</span>
+<span class="sd"> If the buffer is empty, attempts to paginate through the result set for</span>
+<span class="sd"> the next page, and load it into the buffer.</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">None</span>
+
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">buffer</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">all_pages_loaded</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">None</span>
+
+ <span class="n">query_results</span> <span class="o">=</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span><span class="o">.</span><span class="n">getQueryResults</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+ <span class="n">jobId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">job_id</span><span class="p">,</span>
+ <span class="n">pageToken</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">page_token</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">())</span>
+
+ <span class="k">if</span> <span class="s1">'rows'</span> <span class="ow">in</span> <span class="n">query_results</span> <span class="ow">and</span> <span class="n">query_results</span><span class="p">[</span><span class="s1">'rows'</span><span class="p">]:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">page_token</span> <span class="o">=</span> <span class="n">query_results</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'pageToken'</span><span class="p">)</span>
+ <span class="n">fields</span> <span class="o">=</span> <span class="n">query_results</span><span class="p">[</span><span class="s1">'schema'</span><span class="p">][</span><span class="s1">'fields'</span><span class="p">]</span>
+ <span class="n">col_types</span> <span class="o">=</span> <span class="p">[</span><span class="n">field</span><span class="p">[</span><span class="s1">'type'</span><span class="p">]</span> <span class="k">for</span> <span class="n">field</span> <span class="ow">in</span> <span class="n">fields</span><span class="p">]</span>
+ <span class="n">rows</span> <span class="o">=</span> <span class="n">query_results</span><span class="p">[</span><span class="s1">'rows'</span><span class="p">]</span>
+
+ <span class="k">for</span> <span class="n">dict_row</span> <span class="ow">in</span> <span class="n">rows</span><span class="p">:</span>
+ <span class="n">typed_row</span> <span class="o">=</span> <span class="p">([</span>
+ <span class="n">_bq_cast</span><span class="p">(</span><span class="n">vs</span><span class="p">[</span><span class="s1">'v'</span><span class="p">],</span> <span class="n">col_types</span><span class="p">[</span><span class="n">idx</span><span class="p">])</span>
+ <span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">vs</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">dict_row</span><span class="p">[</span><span class="s1">'f'</span><span class="p">])</span>
+ <span class="p">])</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">buffer</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">typed_row</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">page_token</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">all_pages_loaded</span> <span class="o">=</span> <span class="kc">True</span>
+
+ <span class="k">else</span><span class="p">:</span>
+ <span class="c1"># Reset all state since we've exhausted the results.</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">page_token</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">page_token</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="k">return</span> <span class="kc">None</span>
+
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">buffer</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">fetchmany</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">size</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Fetch the next set of rows of a query result, returning a sequence of sequences</span>
+<span class="sd"> (e.g. a list of tuples). An empty sequence is returned when no more rows are</span>
+<span class="sd"> available. The number of rows to fetch per call is specified by the parameter.</span>
+<span class="sd"> If it is not given, the cursor's arraysize determines the number of rows to be</span>
+<span class="sd"> fetched. The method should try to fetch as many rows as indicated by the size</span>
+<span class="sd"> parameter. If this is not possible due to the specified number of rows not being</span>
+<span class="sd"> available, fewer rows may be returned. An :py:class:`~pyhive.exc.Error`</span>
+<span class="sd"> (or subclass) exception is raised if the previous call to</span>
+<span class="sd"> :py:meth:`execute` did not produce any result set or no call was issued yet.</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">size</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">size</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">arraysize</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">size</span><span class="p">):</span>
+ <span class="n">one</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span>
+ <span class="k">if</span> <span class="n">one</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">break</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">result</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">one</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">result</span>
+
+ <span class="k">def</span> <span class="nf">fetchall</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Fetch all (remaining) rows of a query result, returning them as a sequence of</span>
+<span class="sd"> sequences (e.g. a list of tuples).</span>
+<span class="sd"> """</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+ <span class="n">one</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span>
+ <span class="k">if</span> <span class="n">one</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">break</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">result</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">one</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">result</span>
+
+ <span class="k">def</span> <span class="nf">get_arraysize</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">""" Specifies the number of rows to fetch at a time with .fetchmany() """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_buffersize</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">buffersize</span> <span class="k">else</span> <span class="mi">1</span>
+
+ <span class="k">def</span> <span class="nf">set_arraysize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">arraysize</span><span class="p">):</span>
+ <span class="sd">""" Specifies the number of rows to fetch at a time with .fetchmany() """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">buffersize</span> <span class="o">=</span> <span class="n">arraysize</span>
+
+ <span class="n">arraysize</span> <span class="o">=</span> <span class="nb">property</span><span class="p">(</span><span class="n">get_arraysize</span><span class="p">,</span> <span class="n">set_arraysize</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">setinputsizes</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sizes</span><span class="p">):</span>
+ <span class="sd">""" Does nothing by default """</span>
+ <span class="k">pass</span>
+
+ <span class="k">def</span> <span class="nf">setoutputsize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">size</span><span class="p">,</span> <span class="n">column</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">""" Does nothing by default """</span>
+ <span class="k">pass</span>
+
+
+<span class="k">def</span> <span class="nf">_bind_parameters</span><span class="p">(</span><span class="n">operation</span><span class="p">,</span> <span class="n">parameters</span><span class="p">):</span>
+ <span class="sd">""" Helper method that binds parameters to a SQL query. """</span>
+ <span class="c1"># inspired by MySQL Python Connector (conversion.py)</span>
+ <span class="n">string_parameters</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="k">for</span> <span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span> <span class="ow">in</span> <span class="n">iteritems</span><span class="p">(</span><span class="n">parameters</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">value</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">string_parameters</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'NULL'</span>
+ <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">basestring</span><span class="p">):</span>
+ <span class="n">string_parameters</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"'"</span> <span class="o">+</span> <span class="n">_escape</span><span class="p">(</span><span class="n">value</span><span class="p">)</span> <span class="o">+</span> <span class="s2">"'"</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">string_parameters</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">operation</span> <span class="o">%</span> <span class="n">string_parameters</span>
+
+
+<span class="k">def</span> <span class="nf">_escape</span><span class="p">(</span><span class="n">s</span><span class="p">):</span>
+ <span class="sd">""" Helper method that escapes parameters to a SQL query. """</span>
+ <span class="n">e</span> <span class="o">=</span> <span class="n">s</span>
+ <span class="n">e</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="se">\\</span><span class="s1">'</span><span class="p">,</span> <span class="s1">'</span><span class="se">\\\\</span><span class="s1">'</span><span class="p">)</span>
+ <span class="n">e</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">,</span> <span class="s1">'</span><span class="se">\\</span><span class="s1">n'</span><span class="p">)</span>
+ <span class="n">e</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="se">\r</span><span class="s1">'</span><span class="p">,</span> <span class="s1">'</span><span class="se">\\</span><span class="s1">r'</span><span class="p">)</span>
+ <span class="n">e</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"'"</span><span class="p">,</span> <span class="s2">"</span><span class="se">\\</span><span class="s2">'"</span><span class="p">)</span>
+ <span class="n">e</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'"'</span><span class="p">,</span> <span class="s1">'</span><span class="se">\\</span><span class="s1">"'</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">e</span>
+
+
+<span class="k">def</span> <span class="nf">_bq_cast</span><span class="p">(</span><span class="n">string_field</span><span class="p">,</span> <span class="n">bq_type</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Helper method that casts a BigQuery row to the appropriate data types.</span>
+<span class="sd"> This is useful because BigQuery returns all fields as strings.</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">string_field</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">None</span>
+ <span class="k">elif</span> <span class="n">bq_type</span> <span class="o">==</span> <span class="s1">'INTEGER'</span><span class="p">:</span>
+ <span class="k">return</span> <span class="nb">int</span><span class="p">(</span><span class="n">string_field</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="n">bq_type</span> <span class="o">==</span> <span class="s1">'FLOAT'</span> <span class="ow">or</span> <span class="n">bq_type</span> <span class="o">==</span> <span class="s1">'TIMESTAMP'</span><span class="p">:</span>
+ <span class="k">return</span> <span class="nb">float</span><span class="p">(</span><span class="n">string_field</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="n">bq_type</span> <span class="o">==</span> <span class="s1">'BOOLEAN'</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">string_field</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'true'</span><span class="p">,</span> <span class="s1">'false'</span><span class="p">]:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"</span><span class="si">{}</span><span class="s2"> must have value 'true' or 'false'"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">string_field</span><span class="p">))</span>
+ <span class="k">return</span> <span class="n">string_field</span> <span class="o">==</span> <span class="s1">'true'</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">string_field</span>
+
+
+<span class="k">def</span> <span class="nf">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="p">,</span> <span class="n">default_project_id</span><span class="p">,</span> <span class="n">var_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+
+ <span class="k">if</span> <span class="s1">'.'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">table_input</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+ <span class="s1">'Expected target table name in the format of '</span>
+ <span class="s1">'<dataset>.<table>. Got: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">table_input</span><span class="p">))</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">default_project_id</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"INTERNAL: No default project is specified"</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">var_print</span><span class="p">(</span><span class="n">var_name</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">var_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">return</span> <span class="s2">""</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">return</span> <span class="s2">"Format exception for </span><span class="si">{var}</span><span class="s2">: "</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">var</span><span class="o">=</span><span class="n">var_name</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">table_input</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)</span> <span class="o">+</span> <span class="n">table_input</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="s1">':'</span><span class="p">)</span> <span class="o">></span> <span class="mi">3</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">Exception</span><span class="p">((</span><span class="s1">'</span><span class="si">{var}</span><span class="s1">Use either : or . to specify project '</span>
+ <span class="s1">'got </span><span class="si">{input}</span><span class="s1">'</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">var</span><span class="o">=</span><span class="n">var_print</span><span class="p">(</span><span class="n">var_name</span><span class="p">),</span> <span class="nb">input</span><span class="o">=</span><span class="n">table_input</span><span class="p">))</span>
+ <span class="n">cmpt</span> <span class="o">=</span> <span class="n">table_input</span><span class="o">.</span><span class="n">rsplit</span><span class="p">(</span><span class="s1">':'</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+ <span class="n">project_id</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="n">rest</span> <span class="o">=</span> <span class="n">table_input</span>
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">cmpt</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
+ <span class="n">project_id</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="n">rest</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+ <span class="k">elif</span> <span class="nb">len</span><span class="p">(</span><span class="n">cmpt</span><span class="p">)</span> <span class="o">==</span> <span class="mi">2</span> <span class="ow">and</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="s1">':'</span><span class="p">)</span> <span class="o"><=</span> <span class= [...]
+ <span class="k">if</span> <span class="n">cmpt</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">2</span><span class="p">:</span>
+ <span class="n">project_id</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+ <span class="n">rest</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">Exception</span><span class="p">((</span><span class="s1">'</span><span class="si">{var}</span><span class="s1">Expect format of (<project:)<dataset>.<table>, '</span>
+ <span class="s1">'got </span><span class="si">{input}</span><span class="s1">'</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">var</span><span class="o">=</span><span class="n">var_print</span><span class="p">(</span><span class="n">var_name</span><span class="p">),</span> <span class="nb">input</span><span class="o">=</span><span class="n">table_input</span><span class="p">))</span>
+
+ <span class="n">cmpt</span> <span class="o">=</span> <span class="n">rest</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)</span>
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">cmpt</span><span class="p">)</span> <span class="o">==</span> <span class="mi">3</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">project_id</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+ <span class="s2">"</span><span class="si">{var}</span><span class="s2">Use either : or . to specify project"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">var</span><span class="o">=</span><span class="n">var_print</span><span class="p">(</span><span class="n">var_name</span><span class="p">)))</span>
+ <span class="n">project_id</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+ <span class="n">dataset_id</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
+ <span class="n">table_id</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span>
+
+ <span class="k">elif</span> <span class="nb">len</span><span class="p">(</span><span class="n">cmpt</span><span class="p">)</span> <span class="o">==</span> <span class="mi">2</span><span class="p">:</span>
+ <span class="n">dataset_id</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+ <span class="n">table_id</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
+ <span class="p">(</span><span class="s1">'</span><span class="si">{var}</span><span class="s1">Expect format of (<project.|<project:)<dataset>.<table>, '</span>
+ <span class="s1">'got </span><span class="si">{input}</span><span class="s1">'</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">var</span><span class="o">=</span><span class="n">var_print</span><span class="p">(</span><span class="n">var_name</span><span class="p">),</span> <span class="nb">input</span><span class="o">=</span><span class="n">table_input</span><span class="p">))</span>
+
+ <span class="k">if</span> <span class="n">project_id</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">var_name</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">log</span> <span class="o">=</span> <span class="n">LoggingMixin</span><span class="p">()</span><span class="o">.</span><span class="n">log</span>
+ <span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Project not included in </span><span class="si">{var}</span><span class="s1">: </span><span class="si">{input}</span><span class="s1">; '</span>
+ <span class="s1">'using project "</span><span class="si">{project}</span><span class="s1">"'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">var</span><span class="o">=</span><span class="n">var_name</span><span class="p">,</span>
+ <span class="nb">input</span><span class="o">=</span><span class="n">table_input</span><span class="p">,</span>
+ <span class="n">project</span><span class="o">=</span><span class="n">default_project_id</span><span class="p">))</span>
+ <span class="n">project_id</span> <span class="o">=</span> <span class="n">default_project_id</span>
+
+ <span class="k">return</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span>
+
+
+<span class="k">def</span> <span class="nf">_cleanse_time_partitioning</span><span class="p">(</span><span class="n">destination_dataset_table</span><span class="p">,</span> <span class="n">time_partitioning_in</span><span class="p">):</span>
+ <span class="c1"># if it is a partitioned table ($ is in the table name) add partition load option</span>
+
+ <span class="k">if</span> <span class="n">time_partitioning_in</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="n">time_partitioning_in</span> <span class="o">=</span> <span class="p">{}</span>
+
+ <span class="n">time_partitioning_out</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="k">if</span> <span class="n">destination_dataset_table</span> <span class="ow">and</span> <span class="s1">'$'</span> <span class="ow">in</span> <span class="n">destination_dataset_table</span><span class="p">:</span>
+ <span class="n">time_partitioning_out</span><span class="p">[</span><span class="s1">'type'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'DAY'</span>
+ <span class="n">time_partitioning_out</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">time_partitioning_in</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">time_partitioning_out</span>
+
+
+<span class="k">def</span> <span class="nf">_validate_value</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">expected_type</span><span class="p">):</span>
+ <span class="sd">""" function to check expected type and raise</span>
+<span class="sd"> error if type is not correct """</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">expected_type</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">"</span><span class="si">{}</span><span class="s2"> argument must have a type </span><span class="si">{}</span><span class="s2"> not </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">key</span><span class="p">,</span> <span class="n">expected_type</span><span class="p">,</span> <span class="nb">type</span><span class="p">(</span><span class="n">value</span><span class="p">)))</span>
+
+
+<span class="k">def</span> <span class="nf">_api_resource_configs_duplication_check</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">config_dict</span><span class="p">,</span>
+ <span class="n">config_dict_name</span><span class="o">=</span><span class="s1">'api_resource_configs'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">config_dict</span> <span class="ow">and</span> <span class="n">value</span> <span class="o">!=</span> <span class="n">config_dict</span><span class="p">[</span><span class="n">key</span><span class="p">]:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Values of </span><span class="si">{param_name}</span><span class="s2"> param are duplicated. "</span>
+ <span class="s2">"</span><span class="si">{dict_name}</span><span class="s2"> contained </span><span class="si">{param_name}</span><span class="s2"> param "</span>
+ <span class="s2">"in `query` config and </span><span class="si">{param_name}</span><span class="s2"> was also provided "</span>
+ <span class="s2">"with arg to run_query() method. Please remove duplicates."</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">param_name</span><span class="o">=</span><span class="n">key</span><span class="p">,</span> <span class="n">dict_name</span><span class="o">=</span><span class="n">config_dict_name</span><span class="p">))</span>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+ <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/cassandra_hook.html b/1.10.2/_modules/airflow/contrib/hooks/cassandra_hook.html
new file mode 100644
index 0000000..d51e64c
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/cassandra_hook.html
@@ -0,0 +1,416 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.cassandra_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+ <div class="version">
+ 1.10.2
+ </div>
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.cassandra_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.cassandra_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">cassandra.cluster</span> <span class="k">import</span> <span class="n">Cluster</span>
+<span class="kn">from</span> <span class="nn">cassandra.policies</span> <span class="k">import</span> <span class="p">(</span><span class="n">RoundRobinPolicy</span><span class="p">,</span> <span class="n">DCAwareRoundRobinPolicy</span><span class="p">,</span>
+ <span class="n">TokenAwarePolicy</span><span class="p">,</span> <span class="n">WhiteListRoundRobinPolicy</span><span class="p">)</span>
+<span class="kn">from</span> <span class="nn">cassandra.auth</span> <span class="k">import</span> <span class="n">PlainTextAuthProvider</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+
+<div class="viewcode-block" id="CassandraHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.cassandra_hook.CassandraHook">[docs]</a><span class="k">class</span> <span class="nc">CassandraHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Hook used to interact with Cassandra</span>
+
+<span class="sd"> Contact points can be specified as a comma-separated string in the 'hosts'</span>
+<span class="sd"> field of the connection.</span>
+
+<span class="sd"> Port can be specified in the port field of the connection.</span>
+
+<span class="sd"> If SSL is enabled in Cassandra, pass in a dict in the extra field as kwargs for</span>
+<span class="sd"> ``ssl.wrap_socket()``. For example:</span>
+<span class="sd"> {</span>
+<span class="sd"> 'ssl_options' : {</span>
+<span class="sd"> 'ca_certs' : PATH_TO_CA_CERTS</span>
+<span class="sd"> }</span>
+<span class="sd"> }</span>
+
+<span class="sd"> Default load balancing policy is RoundRobinPolicy. To specify a different LB policy:</span>
+<span class="sd"> - DCAwareRoundRobinPolicy</span>
+<span class="sd"> {</span>
+<span class="sd"> 'load_balancing_policy': 'DCAwareRoundRobinPolicy',</span>
+<span class="sd"> 'load_balancing_policy_args': {</span>
+<span class="sd"> 'local_dc': LOCAL_DC_NAME, // optional</span>
+<span class="sd"> 'used_hosts_per_remote_dc': SOME_INT_VALUE, // optional</span>
+<span class="sd"> }</span>
+<span class="sd"> }</span>
+<span class="sd"> - WhiteListRoundRobinPolicy</span>
+<span class="sd"> {</span>
+<span class="sd"> 'load_balancing_policy': 'WhiteListRoundRobinPolicy',</span>
+<span class="sd"> 'load_balancing_policy_args': {</span>
+<span class="sd"> 'hosts': ['HOST1', 'HOST2', 'HOST3']</span>
+<span class="sd"> }</span>
+<span class="sd"> }</span>
+<span class="sd"> - TokenAwarePolicy</span>
+<span class="sd"> {</span>
+<span class="sd"> 'load_balancing_policy': 'TokenAwarePolicy',</span>
+<span class="sd"> 'load_balancing_policy_args': {</span>
+<span class="sd"> 'child_load_balancing_policy': CHILD_POLICY_NAME, // optional</span>
+<span class="sd"> 'child_load_balancing_policy_args': { ... } // optional</span>
+<span class="sd"> }</span>
+<span class="sd"> }</span>
+
+<span class="sd"> For details of the Cluster config, see cassandra.cluster.</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cassandra_conn_id</span><span class="o">=</span><span class="s1">'cassandra_default'</span><span class="p">):</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">cassandra_conn_id</span><span class="p">)</span>
+
+ <span class="n">conn_config</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">:</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s1">'contact_points'</span><span class="p">]</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">','</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">:</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s1">'port'</span><span class="p">]</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">:</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s1">'auth_provider'</span><span class="p">]</span> <span class="o">=</span> <span class="n">PlainTextAuthProvider</span><span class="p">(</span>
+ <span class="n">username</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span> <span class="n">password</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">)</span>
+
+ <span class="n">policy_name</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'load_balancing_policy'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="n">policy_args</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'load_balancing_policy_args'</span><span class="p">,</span> <span class="p">{})</span>
+ <span class="n">lb_policy</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_lb_policy</span><span class="p">(</span><span class="n">policy_name</span><span class="p">,</span> <span class="n">policy_args</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">lb_policy</span><span class="p">:</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s1">'load_balancing_policy'</span><span class="p">]</span> <span class="o">=</span> <span class="n">lb_policy</span>
+
+ <span class="n">cql_version</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'cql_version'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">cql_version</span><span class="p">:</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s1">'cql_version'</span><span class="p">]</span> <span class="o">=</span> <span class="n">cql_version</span>
+
+ <span class="n">ssl_options</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'ssl_options'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">ssl_options</span><span class="p">:</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s1">'ssl_options'</span><span class="p">]</span> <span class="o">=</span> <span class="n">ssl_options</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">cluster</span> <span class="o">=</span> <span class="n">Cluster</span><span class="p">(</span><span class="o">**</span><span class="n">conn_config</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">keyspace</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">schema</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">session</span> <span class="o">=</span> <span class="kc">None</span>
+
+<div class="viewcode-block" id="CassandraHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.cassandra_hook.CassandraHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns a cassandra Session object</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">is_shutdown</span><span class="p">:</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">session</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">keyspace</span><span class="p">)</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span></div>
+
+ <span class="k">def</span> <span class="nf">get_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster</span>
+
+<div class="viewcode-block" id="CassandraHook.shutdown_cluster"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.cassandra_hook.CassandraHook.shutdown_cluster">[docs]</a> <span class="k">def</span> <span class="nf">shutdown_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Closes all sessions and connections associated with this Cluster.</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster</span><span class="o">.</span><span class="n">is_shutdown</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cluster</span><span class="o">.</span><span class="n">shutdown</span><span class="p">()</span></div>
+
+ <span class="nd">@staticmethod</span>
+ <span class="k">def</span> <span class="nf">get_lb_policy</span><span class="p">(</span><span class="n">policy_name</span><span class="p">,</span> <span class="n">policy_args</span><span class="p">):</span>
+ <span class="n">policies</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'RoundRobinPolicy'</span><span class="p">:</span> <span class="n">RoundRobinPolicy</span><span class="p">,</span>
+ <span class="s1">'DCAwareRoundRobinPolicy'</span><span class="p">:</span> <span class="n">DCAwareRoundRobinPolicy</span><span class="p">,</span>
+ <span class="s1">'WhiteListRoundRobinPolicy'</span><span class="p">:</span> <span class="n">WhiteListRoundRobinPolicy</span><span class="p">,</span>
+ <span class="s1">'TokenAwarePolicy'</span><span class="p">:</span> <span class="n">TokenAwarePolicy</span><span class="p">,</span>
+ <span class="p">}</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">policies</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">policy_name</span><span class="p">)</span> <span class="ow">or</span> <span class="n">policy_name</span> <span class="o">==</span> <span class="s1">'RoundRobinPolicy'</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">RoundRobinPolicy</span><span class="p">()</span>
+
+ <span class="k">if</span> <span class="n">policy_name</span> <span class="o">==</span> <span class="s1">'DCAwareRoundRobinPolicy'</span><span class="p">:</span>
+ <span class="n">local_dc</span> <span class="o">=</span> <span class="n">policy_args</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'local_dc'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
+ <span class="n">used_hosts_per_remote_dc</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">policy_args</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'used_hosts_per_remote_dc'</span><span class="p">,</span> <span class="mi">0</span><span class="p">))</span>
+ <span class="k">return</span> <span class="n">DCAwareRoundRobinPolicy</span><span class="p">(</span><span class="n">local_dc</span><span class="p">,</span> <span class="n">used_hosts_per_remote_dc</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">policy_name</span> <span class="o">==</span> <span class="s1">'WhiteListRoundRobinPolicy'</span><span class="p">:</span>
+ <span class="n">hosts</span> <span class="o">=</span> <span class="n">policy_args</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'hosts'</span><span class="p">)</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">hosts</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s1">'Hosts must be specified for WhiteListRoundRobinPolicy'</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">WhiteListRoundRobinPolicy</span><span class="p">(</span><span class="n">hosts</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">policy_name</span> <span class="o">==</span> <span class="s1">'TokenAwarePolicy'</span><span class="p">:</span>
+ <span class="n">allowed_child_policies</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'RoundRobinPolicy'</span><span class="p">,</span>
+ <span class="s1">'DCAwareRoundRobinPolicy'</span><span class="p">,</span>
+ <span class="s1">'WhiteListRoundRobinPolicy'</span><span class="p">,)</span>
+ <span class="n">child_policy_name</span> <span class="o">=</span> <span class="n">policy_args</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'child_load_balancing_policy'</span><span class="p">,</span>
+ <span class="s1">'RoundRobinPolicy'</span><span class="p">)</span>
+ <span class="n">child_policy_args</span> <span class="o">=</span> <span class="n">policy_args</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'child_load_balancing_policy_args'</span><span class="p">,</span> <span class="p">{})</span>
+ <span class="k">if</span> <span class="n">child_policy_name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">allowed_child_policies</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">TokenAwarePolicy</span><span class="p">(</span><span class="n">RoundRobinPolicy</span><span class="p">())</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">child_policy</span> <span class="o">=</span> <span class="n">CassandraHook</span><span class="o">.</span><span class="n">get_lb_policy</span><span class="p">(</span><span class="n">child_policy_name</span><span class="p">,</span>
+ <span class="n">child_policy_args</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">TokenAwarePolicy</span><span class="p">(</span><span class="n">child_policy</span><span class="p">)</span>
+
+<div class="viewcode-block" id="CassandraHook.table_exists"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.cassandra_hook.CassandraHook.table_exists">[docs]</a> <span class="k">def</span> <span class="nf">table_exists</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Checks if a table exists in Cassandra</span>
+
+<span class="sd"> :param table: Target Cassandra table.</span>
+<span class="sd"> Use dot notation to target a specific keyspace.</span>
+<span class="sd"> :type table: string</span>
+<span class="sd"> """</span>
+ <span class="n">keyspace</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">keyspace</span>
+ <span class="k">if</span> <span class="s1">'.'</span> <span class="ow">in</span> <span class="n">table</span><span class="p">:</span>
+ <span class="n">keyspace</span><span class="p">,</span> <span class="n">table</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+ <span class="n">cluster_metadata</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">cluster</span><span class="o">.</span><span class="n">metadata</span>
+ <span class="k">return</span> <span class="p">(</span><span class="n">keyspace</span> <span class="ow">in</span> <span class="n">cluster_metadata</span><span class="o">.</span><span class="n">keyspaces</span> <span class="ow">and</span>
+ <span class="n">table</span> <span class="ow">in</span> <span class="n">cluster_metadata</span><span class="o">.</span><span class="n">keyspaces</span><span class="p">[</span><span class="n">keyspace</span><span class="p">]</span><span class="o">.</span><span class="n">tables</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="CassandraHook.record_exists"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.cassandra_hook.CassandraHook.record_exists">[docs]</a> <span class="k">def</span> <span class="nf">record_exists</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">keys</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Checks if a record exists in Cassandra</span>
+
+<span class="sd"> :param table: Target Cassandra table.</span>
+<span class="sd"> Use dot notation to target a specific keyspace.</span>
+<span class="sd"> :type table: string</span>
+<span class="sd"> :param keys: The keys and their values to check the existence.</span>
+<span class="sd"> :type keys: dict</span>
+<span class="sd"> """</span>
+ <span class="n">keyspace</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">keyspace</span>
+ <span class="k">if</span> <span class="s1">'.'</span> <span class="ow">in</span> <span class="n">table</span><span class="p">:</span>
+ <span class="n">keyspace</span><span class="p">,</span> <span class="n">table</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+ <span class="n">ks</span> <span class="o">=</span> <span class="s2">" AND "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="s2">"</span><span class="si">{}</span><span class="s2">=%(</span><span class="si">{}</span><span class="s2">)s"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">key</span><span class="p">)</spa [...]
+ <span class="n">cql</span> <span class="o">=</span> <span class="s2">"SELECT * FROM </span><span class="si">{keyspace}</span><span class="s2">.</span><span class="si">{table}</span><span class="s2"> WHERE </span><span class="si">{keys}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">keyspace</span><span class="o">=</span><span class="n">keyspace</span><span class="p">,</span> <span class="n">table</span><span class="o">=</span><span class="n">table</span><span class="p">,</span> <span class="n">keys</span><span class="o">=</span><span class="n">ks</span><span class="p">)</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">rs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">cql</span><span class="p">,</span> <span class="n">keys</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">rs</span><span class="o">.</span><span class="n">one</span><span class="p">()</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+ <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
+ <span class="k">return</span> <span class="kc">False</span></div></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+ <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/cloudant_hook.html b/1.10.2/_modules/airflow/contrib/hooks/cloudant_hook.html
new file mode 100644
index 0000000..b28e966
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/cloudant_hook.html
@@ -0,0 +1,302 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.cloudant_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+ <div class="version">
+ 1.10.2
+ </div>
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.cloudant_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.cloudant_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="k">import</span> <span class="n">unicode</span>
+
+<span class="kn">import</span> <span class="nn">cloudant</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+
+<div class="viewcode-block" id="CloudantHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.cloudant_hook.CloudantHook">[docs]</a><span class="k">class</span> <span class="nc">CloudantHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+ <span class="sd">"""Interact with Cloudant.</span>
+
+<span class="sd"> This class is a thin wrapper around the cloudant python library. See the</span>
+<span class="sd"> documentation `here <https://github.com/cloudant-labs/cloudant-python>`_.</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cloudant_conn_id</span><span class="o">=</span><span class="s1">'cloudant_default'</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">CloudantHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="s1">'cloudant'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cloudant_conn_id</span> <span class="o">=</span> <span class="n">cloudant_conn_id</span>
+
+ <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">_str</span><span class="p">(</span><span class="n">s</span><span class="p">):</span>
+ <span class="c1"># cloudant-python doesn't support unicode.</span>
+ <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="n">unicode</span><span class="p">):</span>
+ <span class="n">log</span> <span class="o">=</span> <span class="n">LoggingMixin</span><span class="p">()</span><span class="o">.</span><span class="n">log</span>
+ <span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
+ <span class="s1">'cloudant-python does not support unicode. Encoding </span><span class="si">%s</span><span class="s1"> as '</span>
+ <span class="s1">'ascii using "ignore".'</span><span class="p">,</span> <span class="n">s</span>
+ <span class="p">)</span>
+ <span class="k">return</span> <span class="n">s</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'ascii'</span><span class="p">,</span> <span class="s1">'ignore'</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">s</span>
+
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cloudant_conn_id</span><span class="p">)</span>
+
+ <span class="k">for</span> <span class="n">conn_param</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'host'</span><span class="p">,</span> <span class="s1">'password'</span><span class="p">,</span> <span class="s1">'schema'</span><span class="p">]:</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">conn</span><span class="p">,</span> <span class="n">conn_param</span><span class="p">)</span> <span class="ow">or</span> <span class="ow">not</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">conn</span><span class="p">,</span> <span class="n">conn_param</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="s1">'missing connection parameter </span><span class="si">{0}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">conn_param</span><span class="p">)</span>
+ <span class="p">)</span>
+
+ <span class="c1"># In the connection form:</span>
+ <span class="c1"># - 'host' is renamed to 'Account'</span>
+ <span class="c1"># - 'login' is renamed 'Username (or API Key)'</span>
+ <span class="c1"># - 'schema' is renamed to 'Database'</span>
+ <span class="c1">#</span>
+ <span class="c1"># So, use the 'host' attribute as the account name, and, if login is</span>
+ <span class="c1"># defined, use that as the username.</span>
+ <span class="n">account</span> <span class="o">=</span> <span class="n">cloudant</span><span class="o">.</span><span class="n">Account</span><span class="p">(</span><span class="n">_str</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">))</span>
+
+ <span class="n">username</span> <span class="o">=</span> <span class="n">_str</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span> <span class="ow">or</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">)</span>
+
+ <span class="n">account</span><span class="o">.</span><span class="n">login</span><span class="p">(</span>
+ <span class="n">username</span><span class="p">,</span>
+ <span class="n">_str</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">))</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
+
+ <span class="k">return</span> <span class="n">account</span><span class="o">.</span><span class="n">database</span><span class="p">(</span><span class="n">_str</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">schema</span><span class="p">))</span>
+
+<div class="viewcode-block" id="CloudantHook.db"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.cloudant_hook.CloudantHook.db">[docs]</a> <span class="k">def</span> <span class="nf">db</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""Returns the Database object for this hook.</span>
+
+<span class="sd"> See the documentation for cloudant-python here</span>
+<span class="sd"> https://github.com/cloudant-labs/cloudant-python.</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span></div></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+ <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/databricks_hook.html b/1.10.2/_modules/airflow/contrib/hooks/databricks_hook.html
new file mode 100644
index 0000000..537515a
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/databricks_hook.html
@@ -0,0 +1,495 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.databricks_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+ <div class="version">
+ 1.10.2
+ </div>
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.databricks_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.databricks_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+<span class="kn">import</span> <span class="nn">requests</span>
+
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">__version__</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">requests</span> <span class="k">import</span> <span class="n">exceptions</span> <span class="k">as</span> <span class="n">requests_exceptions</span>
+<span class="kn">from</span> <span class="nn">requests.auth</span> <span class="k">import</span> <span class="n">AuthBase</span>
+<span class="kn">from</span> <span class="nn">time</span> <span class="k">import</span> <span class="n">sleep</span>
+
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+<span class="k">try</span><span class="p">:</span>
+ <span class="kn">from</span> <span class="nn">urllib</span> <span class="k">import</span> <span class="n">parse</span> <span class="k">as</span> <span class="n">urlparse</span>
+<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
+ <span class="kn">import</span> <span class="nn">urlparse</span>
+
+<span class="n">RESTART_CLUSTER_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s2">"POST"</span><span class="p">,</span> <span class="s2">"api/2.0/clusters/restart"</span><span class="p">)</span>
+<span class="n">START_CLUSTER_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s2">"POST"</span><span class="p">,</span> <span class="s2">"api/2.0/clusters/start"</span><span class="p">)</span>
+<span class="n">TERMINATE_CLUSTER_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s2">"POST"</span><span class="p">,</span> <span class="s2">"api/2.0/clusters/delete"</span><span class="p">)</span>
+
+<span class="n">RUN_NOW_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'POST'</span><span class="p">,</span> <span class="s1">'api/2.0/jobs/run-now'</span><span class="p">)</span>
+<span class="n">SUBMIT_RUN_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'POST'</span><span class="p">,</span> <span class="s1">'api/2.0/jobs/runs/submit'</span><span class="p">)</span>
+<span class="n">GET_RUN_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'GET'</span><span class="p">,</span> <span class="s1">'api/2.0/jobs/runs/get'</span><span class="p">)</span>
+<span class="n">CANCEL_RUN_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'POST'</span><span class="p">,</span> <span class="s1">'api/2.0/jobs/runs/cancel'</span><span class="p">)</span>
+<span class="n">USER_AGENT_HEADER</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'user-agent'</span><span class="p">:</span> <span class="s1">'airflow-</span><span class="si">{v}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">v</span><span class="o">=</span><span class="n">__version__</span><span class="p">)}</span>
+
+
+<div class="viewcode-block" id="DatabricksHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.databricks_hook.DatabricksHook">[docs]</a><span class="k">class</span> <span class="nc">DatabricksHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with Databricks.</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">databricks_conn_id</span><span class="o">=</span><span class="s1">'databricks_default'</span><span class="p">,</span>
+ <span class="n">timeout_seconds</span><span class="o">=</span><span class="mi">180</span><span class="p">,</span>
+ <span class="n">retry_limit</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span>
+ <span class="n">retry_delay</span><span class="o">=</span><span class="mf">1.0</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> :param databricks_conn_id: The name of the databricks connection to use.</span>
+<span class="sd"> :type databricks_conn_id: string</span>
+<span class="sd"> :param timeout_seconds: The amount of time in seconds the requests library</span>
+<span class="sd"> will wait before timing-out.</span>
+<span class="sd"> :type timeout_seconds: int</span>
+<span class="sd"> :param retry_limit: The number of times to retry the connection in case of</span>
+<span class="sd"> service outages.</span>
+<span class="sd"> :type retry_limit: int</span>
+<span class="sd"> :param retry_delay: The number of seconds to wait between retries (it</span>
+<span class="sd"> might be a floating point number).</span>
+<span class="sd"> :type retry_delay: float</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn_id</span> <span class="o">=</span> <span class="n">databricks_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">databricks_conn_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">timeout_seconds</span> <span class="o">=</span> <span class="n">timeout_seconds</span>
+ <span class="k">if</span> <span class="n">retry_limit</span> <span class="o"><</span> <span class="mi">1</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'Retry limit must be greater than equal to 1'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">retry_limit</span> <span class="o">=</span> <span class="n">retry_limit</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">retry_delay</span> <span class="o">=</span> <span class="n">retry_delay</span>
+
+ <span class="nd">@staticmethod</span>
+ <span class="k">def</span> <span class="nf">_parse_host</span><span class="p">(</span><span class="n">host</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> The purpose of this function is to be robust to improper connections</span>
+<span class="sd"> settings provided by users, specifically in the host field.</span>
+
+
+<span class="sd"> For example -- when users supply ``https://xx.cloud.databricks.com`` as the</span>
+<span class="sd"> host, we must strip out the protocol to get the host.</span>
+<span class="sd"> >>> h = DatabricksHook()</span>
+<span class="sd"> >>> assert h._parse_host('https://xx.cloud.databricks.com') == \</span>
+<span class="sd"> 'xx.cloud.databricks.com'</span>
+
+<span class="sd"> In the case where users supply the correct ``xx.cloud.databricks.com`` as the</span>
+<span class="sd"> host, this function is a no-op.</span>
+<span class="sd"> >>> assert h._parse_host('xx.cloud.databricks.com') == 'xx.cloud.databricks.com'</span>
+<span class="sd"> """</span>
+ <span class="n">urlparse_host</span> <span class="o">=</span> <span class="n">urlparse</span><span class="o">.</span><span class="n">urlparse</span><span class="p">(</span><span class="n">host</span><span class="p">)</span><span class="o">.</span><span class="n">hostname</span>
+ <span class="k">if</span> <span class="n">urlparse_host</span><span class="p">:</span>
+ <span class="c1"># In this case, host = https://xx.cloud.databricks.com</span>
+ <span class="k">return</span> <span class="n">urlparse_host</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="c1"># In this case, host = xx.cloud.databricks.com</span>
+ <span class="k">return</span> <span class="n">host</span>
+
+ <span class="k">def</span> <span class="nf">_do_api_call</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">endpoint_info</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Utility function to perform an API call with retries</span>
+<span class="sd"> :param endpoint_info: Tuple of method and endpoint</span>
+<span class="sd"> :type endpoint_info: (string, string)</span>
+<span class="sd"> :param json: Parameters for this API call.</span>
+<span class="sd"> :type json: dict</span>
+<span class="sd"> :return: If the api call returns a OK status code,</span>
+<span class="sd"> this function returns the response in JSON. Otherwise,</span>
+<span class="sd"> we throw an AirflowException.</span>
+<span class="sd"> :rtype: dict</span>
+<span class="sd"> """</span>
+ <span class="n">method</span><span class="p">,</span> <span class="n">endpoint</span> <span class="o">=</span> <span class="n">endpoint_info</span>
+ <span class="n">url</span> <span class="o">=</span> <span class="s1">'https://</span><span class="si">{host}</span><span class="s1">/</span><span class="si">{endpoint}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">host</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_parse_host</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn</span><span class="o">.</span><span class="n">host</span><span class="p">),</span>
+ <span class="n">endpoint</span><span class="o">=</span><span class="n">endpoint</span><span class="p">)</span>
+ <span class="k">if</span> <span class="s1">'token'</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Using token auth.'</span><span class="p">)</span>
+ <span class="n">auth</span> <span class="o">=</span> <span class="n">_TokenAuth</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s1">'token'</span><span class="p">])</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Using basic auth.'</span><span class="p">)</span>
+ <span class="n">auth</span> <span class="o">=</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn</span><span class="o">.</span><span class="n">password</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">method</span> <span class="o">==</span> <span class="s1">'GET'</span><span class="p">:</span>
+ <span class="n">request_func</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">get</span>
+ <span class="k">elif</span> <span class="n">method</span> <span class="o">==</span> <span class="s1">'POST'</span><span class="p">:</span>
+ <span class="n">request_func</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">post</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Unexpected HTTP Method: '</span> <span class="o">+</span> <span class="n">method</span><span class="p">)</span>
+
+ <span class="n">attempt_num</span> <span class="o">=</span> <span class="mi">1</span>
+ <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="n">request_func</span><span class="p">(</span>
+ <span class="n">url</span><span class="p">,</span>
+ <span class="n">json</span><span class="o">=</span><span class="n">json</span><span class="p">,</span>
+ <span class="n">auth</span><span class="o">=</span><span class="n">auth</span><span class="p">,</span>
+ <span class="n">headers</span><span class="o">=</span><span class="n">USER_AGENT_HEADER</span><span class="p">,</span>
+ <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout_seconds</span><span class="p">)</span>
+ <span class="n">response</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
+ <span class="k">except</span> <span class="n">requests_exceptions</span><span class="o">.</span><span class="n">RequestException</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">_retryable_error</span><span class="p">(</span><span class="n">e</span><span class="p">):</span>
+ <span class="c1"># In this case, the user probably made a mistake.</span>
+ <span class="c1"># Don't retry.</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Response: </span><span class="si">{0}</span><span class="s1">, Status Code: </span><span class="si">{1}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="o">.</span><span class="n">content</span><span class="p">,</span> <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="o">.</span><span class="n">status_code</span><span class="p">))</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">_log_request_error</span><span class="p">(</span><span class="n">attempt_num</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">attempt_num</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">retry_limit</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">((</span><span class="s1">'API requests to Databricks failed </span><span class="si">{}</span><span class="s1"> times. '</span> <span class="o">+</span>
+ <span class="s1">'Giving up.'</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">retry_limit</span><span class="p">))</span>
+
+ <span class="n">attempt_num</span> <span class="o">+=</span> <span class="mi">1</span>
+ <span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">retry_delay</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">_log_request_error</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attempt_num</span><span class="p">,</span> <span class="n">error</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
+ <span class="s1">'Attempt </span><span class="si">%s</span><span class="s1"> API Request to Databricks failed with reason: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
+ <span class="n">attempt_num</span><span class="p">,</span> <span class="n">error</span>
+ <span class="p">)</span>
+
+<div class="viewcode-block" id="DatabricksHook.run_now"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.databricks_hook.DatabricksHook.run_now">[docs]</a> <span class="k">def</span> <span class="nf">run_now</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Utility function to call the ``api/2.0/jobs/run-now`` endpoint.</span>
+
+<span class="sd"> :param json: The data used in the body of the request to the ``run-now`` endpoint.</span>
+<span class="sd"> :type json: dict</span>
+<span class="sd"> :return: the run_id as a string</span>
+<span class="sd"> :rtype: string</span>
+<span class="sd"> """</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">RUN_NOW_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">response</span><span class="p">[</span><span class="s1">'run_id'</span><span class="p">]</span></div>
+
+<div class="viewcode-block" id="DatabricksHook.submit_run"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.databricks_hook.DatabricksHook.submit_run">[docs]</a> <span class="k">def</span> <span class="nf">submit_run</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Utility function to call the ``api/2.0/jobs/runs/submit`` endpoint.</span>
+
+<span class="sd"> :param json: The data used in the body of the request to the ``submit`` endpoint.</span>
+<span class="sd"> :type json: dict</span>
+<span class="sd"> :return: the run_id as a string</span>
+<span class="sd"> :rtype: string</span>
+<span class="sd"> """</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">SUBMIT_RUN_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">response</span><span class="p">[</span><span class="s1">'run_id'</span><span class="p">]</span></div>
+
+ <span class="k">def</span> <span class="nf">get_run_page_url</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">run_id</span><span class="p">):</span>
+ <span class="n">json</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'run_id'</span><span class="p">:</span> <span class="n">run_id</span><span class="p">}</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">GET_RUN_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">response</span><span class="p">[</span><span class="s1">'run_page_url'</span><span class="p">]</span>
+
+ <span class="k">def</span> <span class="nf">get_run_state</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">run_id</span><span class="p">):</span>
+ <span class="n">json</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'run_id'</span><span class="p">:</span> <span class="n">run_id</span><span class="p">}</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">GET_RUN_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span>
+ <span class="n">state</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">'state'</span><span class="p">]</span>
+ <span class="n">life_cycle_state</span> <span class="o">=</span> <span class="n">state</span><span class="p">[</span><span class="s1">'life_cycle_state'</span><span class="p">]</span>
+ <span class="c1"># result_state may not be in the state if not terminal</span>
+ <span class="n">result_state</span> <span class="o">=</span> <span class="n">state</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'result_state'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="n">state_message</span> <span class="o">=</span> <span class="n">state</span><span class="p">[</span><span class="s1">'state_message'</span><span class="p">]</span>
+ <span class="k">return</span> <span class="n">RunState</span><span class="p">(</span><span class="n">life_cycle_state</span><span class="p">,</span> <span class="n">result_state</span><span class="p">,</span> <span class="n">state_message</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">cancel_run</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">run_id</span><span class="p">):</span>
+ <span class="n">json</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'run_id'</span><span class="p">:</span> <span class="n">run_id</span><span class="p">}</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">CANCEL_RUN_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">restart_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">RESTART_CLUSTER_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">start_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">START_CLUSTER_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">terminate_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">TERMINATE_CLUSTER_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span></div>
+
+
+<span class="k">def</span> <span class="nf">_retryable_error</span><span class="p">(</span><span class="n">exception</span><span class="p">):</span>
+ <span class="k">return</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">exception</span><span class="p">,</span> <span class="n">requests_exceptions</span><span class="o">.</span><span class="n">ConnectionError</span><span class="p">)</span> \
+ <span class="ow">or</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">exception</span><span class="p">,</span> <span class="n">requests_exceptions</span><span class="o">.</span><span class="n">Timeout</span><span class="p">)</span> \
+ <span class="ow">or</span> <span class="n">exception</span><span class="o">.</span><span class="n">response</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">exception</span><span class="o">.</span><span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">>=</span> <span class="mi">500</span>
+
+
+<span class="n">RUN_LIFE_CYCLE_STATES</span> <span class="o">=</span> <span class="p">[</span>
+ <span class="s1">'PENDING'</span><span class="p">,</span>
+ <span class="s1">'RUNNING'</span><span class="p">,</span>
+ <span class="s1">'TERMINATING'</span><span class="p">,</span>
+ <span class="s1">'TERMINATED'</span><span class="p">,</span>
+ <span class="s1">'SKIPPED'</span><span class="p">,</span>
+ <span class="s1">'INTERNAL_ERROR'</span>
+<span class="p">]</span>
+
+
+<span class="k">class</span> <span class="nc">RunState</span><span class="p">:</span>
+ <span class="sd">"""</span>
+<span class="sd"> Utility class for the run state concept of Databricks runs.</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">life_cycle_state</span><span class="p">,</span> <span class="n">result_state</span><span class="p">,</span> <span class="n">state_message</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">life_cycle_state</span> <span class="o">=</span> <span class="n">life_cycle_state</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">result_state</span> <span class="o">=</span> <span class="n">result_state</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">state_message</span> <span class="o">=</span> <span class="n">state_message</span>
+
+ <span class="nd">@property</span>
+ <span class="k">def</span> <span class="nf">is_terminal</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">life_cycle_state</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">RUN_LIFE_CYCLE_STATES</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+ <span class="p">(</span><span class="s1">'Unexpected life cycle state: </span><span class="si">{}</span><span class="s1">: If the state has '</span>
+ <span class="s1">'been introduced recently, please check the Databricks user '</span>
+ <span class="s1">'guide for troubleshooting information'</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">life_cycle_state</span><span class="p">))</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">life_cycle_state</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'TERMINATED'</span><span class="p">,</span> <span class="s1">'SKIPPED'</span><span class="p">,</span> <span class="s1">'INTERNAL_ERROR'</span><span class="p">)</span>
+
+ <span class="nd">@property</span>
+ <span class="k">def</span> <span class="nf">is_successful</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_state</span> <span class="o">==</span> <span class="s1">'SUCCESS'</span>
+
+ <span class="k">def</span> <span class="nf">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">life_cycle_state</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">life_cycle_state</span> <span class="ow">and</span> \
+ <span class="bp">self</span><span class="o">.</span><span class="n">result_state</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">result_state</span> <span class="ow">and</span> \
+ <span class="bp">self</span><span class="o">.</span><span class="n">state_message</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">state_message</span>
+
+ <span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">_TokenAuth</span><span class="p">(</span><span class="n">AuthBase</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Helper class for requests Auth field. AuthBase requires you to implement the __call__</span>
+<span class="sd"> magic function.</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">token</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">token</span> <span class="o">=</span> <span class="n">token</span>
+
+ <span class="k">def</span> <span class="nf">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">r</span><span class="p">):</span>
+ <span class="n">r</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="s1">'Authorization'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'Bearer '</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">token</span>
+ <span class="k">return</span> <span class="n">r</span>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+ <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/datadog_hook.html b/1.10.2/_modules/airflow/contrib/hooks/datadog_hook.html
new file mode 100644
index 0000000..a2b4683
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/datadog_hook.html
@@ -0,0 +1,361 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.datadog_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+ <div class="version">
+ 1.10.2
+ </div>
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.datadog_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.datadog_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">time</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">datadog</span> <span class="k">import</span> <span class="n">initialize</span><span class="p">,</span> <span class="n">api</span>
+
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+
+<div class="viewcode-block" id="DatadogHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.datadog_hook.DatadogHook">[docs]</a><span class="k">class</span> <span class="nc">DatadogHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Uses datadog API to send metrics of practically anything measurable,</span>
+<span class="sd"> so it's possible to track # of db records inserted/deleted, records read</span>
+<span class="sd"> from file and many other useful metrics.</span>
+
+<span class="sd"> Depends on the datadog API, which has to be deployed on the same server where</span>
+<span class="sd"> Airflow runs.</span>
+
+<span class="sd"> :param datadog_conn_id: The connection to datadog, containing metadata for api keys.</span>
+<span class="sd"> :param datadog_conn_id: string</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">datadog_conn_id</span><span class="o">=</span><span class="s1">'datadog_default'</span><span class="p">):</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">datadog_conn_id</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">api_key</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'api_key'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">app_key</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'app_key'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">source_type_name</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'source_type_name'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+
+ <span class="c1"># If the host is populated, it will use that hostname instead.</span>
+ <span class="c1"># for all metric submissions.</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">host</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">api_key</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"api_key must be specified in the "</span>
+ <span class="s2">"Datadog connection details"</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">app_key</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"app_key must be specified in the "</span>
+ <span class="s2">"Datadog connection details"</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Setting up api keys for Datadog"</span><span class="p">)</span>
+ <span class="n">options</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'api_key'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">api_key</span><span class="p">,</span>
+ <span class="s1">'app_key'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">app_key</span>
+ <span class="p">}</span>
+ <span class="n">initialize</span><span class="p">(</span><span class="o">**</span><span class="n">options</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">validate_response</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">response</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s1">'status'</span><span class="p">]</span> <span class="o">!=</span> <span class="s1">'ok'</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Datadog returned: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">response</span><span class="p">)</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Error status received from Datadog"</span><span class="p">)</span>
+
+<div class="viewcode-block" id="DatadogHook.send_metric"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.datadog_hook.DatadogHook.send_metric">[docs]</a> <span class="k">def</span> <span class="nf">send_metric</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">metric_name</span><span class="p">,</span> <span class="n">datapoint</span><span class="p">,</span> <span class="n">tags</span><span class="o">=</span><s [...]
+ <span class="sd">"""</span>
+<span class="sd"> Sends a single datapoint metric to DataDog</span>
+
+<span class="sd"> :param metric_name: The name of the metric</span>
+<span class="sd"> :type metric_name: string</span>
+<span class="sd"> :param datapoint: A single integer or float related to the metric</span>
+<span class="sd"> :type datapoint: integer or float</span>
+<span class="sd"> :param tags: A list of tags associated with the metric</span>
+<span class="sd"> :type tags: list</span>
+<span class="sd"> """</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="n">api</span><span class="o">.</span><span class="n">Metric</span><span class="o">.</span><span class="n">send</span><span class="p">(</span>
+ <span class="n">metric</span><span class="o">=</span><span class="n">metric_name</span><span class="p">,</span>
+ <span class="n">points</span><span class="o">=</span><span class="n">datapoint</span><span class="p">,</span>
+ <span class="n">host</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">host</span><span class="p">,</span>
+ <span class="n">tags</span><span class="o">=</span><span class="n">tags</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">validate_response</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">response</span></div>
+
+<div class="viewcode-block" id="DatadogHook.query_metric"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.datadog_hook.DatadogHook.query_metric">[docs]</a> <span class="k">def</span> <span class="nf">query_metric</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">query</span><span class="p">,</span>
+ <span class="n">from_seconds_ago</span><span class="p">,</span>
+ <span class="n">to_seconds_ago</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Queries datadog for a specific metric, potentially with some</span>
+<span class="sd"> function applied to it and returns the results.</span>
+
+<span class="sd"> :param query: The datadog query to execute (see datadog docs)</span>
+<span class="sd"> :type query: string</span>
+<span class="sd"> :param from_seconds_ago: How many seconds ago to start querying for.</span>
+<span class="sd"> :type from_seconds_ago: int</span>
+<span class="sd"> :param to_seconds_ago: Up to how many seconds ago to query for.</span>
+<span class="sd"> :type to_seconds_ago: int</span>
+<span class="sd"> """</span>
+ <span class="n">now</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">())</span>
+
+ <span class="n">response</span> <span class="o">=</span> <span class="n">api</span><span class="o">.</span><span class="n">Metric</span><span class="o">.</span><span class="n">query</span><span class="p">(</span>
+ <span class="n">start</span><span class="o">=</span><span class="n">now</span> <span class="o">-</span> <span class="n">from_seconds_ago</span><span class="p">,</span>
+ <span class="n">end</span><span class="o">=</span><span class="n">now</span> <span class="o">-</span> <span class="n">to_seconds_ago</span><span class="p">,</span>
+ <span class="n">query</span><span class="o">=</span><span class="n">query</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">validate_response</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">response</span></div>
+
+<div class="viewcode-block" id="DatadogHook.post_event"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.datadog_hook.DatadogHook.post_event">[docs]</a> <span class="k">def</span> <span class="nf">post_event</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">title</span><span class="p">,</span> <span class="n">text</span><span class="p">,</span> <span class="n">tags</span><span class="o">=</span><span class="kc" [...]
+ <span class="sd">"""</span>
+<span class="sd"> Posts an event to datadog (processing finished, potentially alerts, other issues)</span>
+<span class="sd"> Think about this as a means to maintain persistence of alerts, rather than</span>
+<span class="sd"> alerting itself.</span>
+
+<span class="sd"> :param title: The title of the event</span>
+<span class="sd"> :type title: string</span>
+<span class="sd"> :param text: The body of the event (more information)</span>
+<span class="sd"> :type text: string</span>
+<span class="sd"> :param tags: List of string tags to apply to the event</span>
+<span class="sd"> :type tags: list</span>
+<span class="sd"> :param alert_type: The alert type for the event, one of</span>
+<span class="sd"> ["error", "warning", "info", "success"]</span>
+<span class="sd"> :type alert_type: string</span>
+<span class="sd"> :param aggregation_key: Key that can be used to aggregate this event in a stream</span>
+<span class="sd"> :type aggregation_key: string</span>
+<span class="sd"> """</span>
+ <span class="n">response</span> <span class="o">=</span> <span class="n">api</span><span class="o">.</span><span class="n">Event</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
+ <span class="n">title</span><span class="o">=</span><span class="n">title</span><span class="p">,</span>
+ <span class="n">text</span><span class="o">=</span><span class="n">text</span><span class="p">,</span>
+ <span class="n">host</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">host</span><span class="p">,</span>
+ <span class="n">tags</span><span class="o">=</span><span class="n">tags</span><span class="p">,</span>
+ <span class="n">alert_type</span><span class="o">=</span><span class="n">alert_type</span><span class="p">,</span>
+ <span class="n">aggregation_key</span><span class="o">=</span><span class="n">aggregation_key</span><span class="p">,</span>
+ <span class="n">source_type_name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">source_type_name</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">validate_response</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">response</span></div></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+ <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/datastore_hook.html b/1.10.2/_modules/airflow/contrib/hooks/datastore_hook.html
new file mode 100644
index 0000000..3918dc5
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/datastore_hook.html
@@ -0,0 +1,425 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.datastore_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+ <div class="version">
+ 1.10.2
+ </div>
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.datastore_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.datastore_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+
+<span class="kn">import</span> <span class="nn">time</span>
+<span class="kn">from</span> <span class="nn">googleapiclient.discovery</span> <span class="k">import</span> <span class="n">build</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="k">import</span> <span class="n">GoogleCloudBaseHook</span>
+
+
+<div class="viewcode-block" id="DatastoreHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook">[docs]</a><span class="k">class</span> <span class="nc">DatastoreHook</span><span class="p">(</span><span class="n">GoogleCloudBaseHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Interact with Google Cloud Datastore. This hook uses the Google Cloud Platform</span>
+<span class="sd"> connection.</span>
+
+<span class="sd"> This object is not threads safe. If you want to make multiple requests</span>
+<span class="sd"> simultaneously, you will need to create a hook per thread.</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">datastore_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_datastore_default'</span><span class="p">,</span>
+ <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">DatastoreHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">datastore_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">connection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">admin_connection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">(</span><span class="s1">'v1beta1'</span><span class="p">)</span>
+
+<div class="viewcode-block" id="DatastoreHook.get_conn"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">version</span><span class="o">=</span><span class="s1">'v1'</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns a Google Cloud Datastore service object.</span>
+<span class="sd"> """</span>
+ <span class="n">http_authorized</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">build</span><span class="p">(</span>
+ <span class="s1">'datastore'</span><span class="p">,</span> <span class="n">version</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">,</span> <span class="n">cache_discovery</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.allocate_ids"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.allocate_ids">[docs]</a> <span class="k">def</span> <span class="nf">allocate_ids</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">partialKeys</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Allocate IDs for incomplete keys.</span>
+<span class="sd"> see https://cloud.google.com/datastore/docs/reference/rest/v1/projects/allocateIds</span>
+
+<span class="sd"> :param partialKeys: a list of partial keys</span>
+<span class="sd"> :return: a list of full keys.</span>
+<span class="sd"> """</span>
+ <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">allocateIds</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="p">{</span><span class="s1">'keys'</span><span class="p">:</span> <span class="n">partialKeys</span><span class="p">}</span>
+ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">resp</span><span class="p">[</span><span class="s1">'keys'</span><span class="p">]</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.begin_transaction"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.begin_transaction">[docs]</a> <span class="k">def</span> <span class="nf">begin_transaction</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Get a new transaction handle</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> https://cloud.google.com/datastore/docs/reference/rest/v1/projects/beginTransaction</span>
+
+<span class="sd"> :return: a transaction handle</span>
+<span class="sd"> """</span>
+ <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">beginTransaction</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="p">{})</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">resp</span><span class="p">[</span><span class="s1">'transaction'</span><span class="p">]</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.commit"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.commit">[docs]</a> <span class="k">def</span> <span class="nf">commit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">body</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Commit a transaction, optionally creating, deleting or modifying some entities.</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> https://cloud.google.com/datastore/docs/reference/rest/v1/projects/commit</span>
+
+<span class="sd"> :param body: the body of the commit request</span>
+<span class="sd"> :return: the response body of the commit request</span>
+<span class="sd"> """</span>
+ <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">commit</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">resp</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.lookup"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.lookup">[docs]</a> <span class="k">def</span> <span class="nf">lookup</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">keys</span><span class="p">,</span> <span class="n">read_consistency</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <sp [...]
+ <span class="sd">"""</span>
+<span class="sd"> Lookup some entities by key</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> https://cloud.google.com/datastore/docs/reference/rest/v1/projects/lookup</span>
+
+<span class="sd"> :param keys: the keys to lookup</span>
+<span class="sd"> :param read_consistency: the read consistency to use. default, strong or eventual.</span>
+<span class="sd"> Cannot be used with a transaction.</span>
+<span class="sd"> :param transaction: the transaction to use, if any.</span>
+<span class="sd"> :return: the response body of the lookup request.</span>
+<span class="sd"> """</span>
+ <span class="n">body</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'keys'</span><span class="p">:</span> <span class="n">keys</span><span class="p">}</span>
+ <span class="k">if</span> <span class="n">read_consistency</span><span class="p">:</span>
+ <span class="n">body</span><span class="p">[</span><span class="s1">'readConsistency'</span><span class="p">]</span> <span class="o">=</span> <span class="n">read_consistency</span>
+ <span class="k">if</span> <span class="n">transaction</span><span class="p">:</span>
+ <span class="n">body</span><span class="p">[</span><span class="s1">'transaction'</span><span class="p">]</span> <span class="o">=</span> <span class="n">transaction</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">lookup</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.rollback"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.rollback">[docs]</a> <span class="k">def</span> <span class="nf">rollback</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">transaction</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Roll back a transaction</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> https://cloud.google.com/datastore/docs/reference/rest/v1/projects/rollback</span>
+
+<span class="sd"> :param transaction: the transaction to roll back</span>
+<span class="sd"> """</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">rollback</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="p">{</span><span class="s1">'transaction'</span><span class="p">:</span> <span class="n">transaction</span><span class="p">})</span>\
+ <span class="o">.</span><span class="n">execute</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.run_query"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.run_query">[docs]</a> <span class="k">def</span> <span class="nf">run_query</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">body</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Run a query for entities.</span>
+
+<span class="sd"> .. seealso::</span>
+<span class="sd"> https://cloud.google.com/datastore/docs/reference/rest/v1/projects/runQuery</span>
+
+<span class="sd"> :param body: the body of the query request</span>
+<span class="sd"> :return: the batch of query results.</span>
+<span class="sd"> """</span>
+ <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">runQuery</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">resp</span><span class="p">[</span><span class="s1">'batch'</span><span class="p">]</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.get_operation"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.get_operation">[docs]</a> <span class="k">def</span> <span class="nf">get_operation</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Gets the latest state of a long-running operation</span>
+
+<span class="sd"> :param name: the name of the operation resource</span>
+<span class="sd"> """</span>
+ <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">operations</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="n">name</span><span class="p">)</span><span cl [...]
+ <span class="k">return</span> <span class="n">resp</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.delete_operation"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.delete_operation">[docs]</a> <span class="k">def</span> <span class="nf">delete_operation</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Deletes the long-running operation</span>
+
+<span class="sd"> :param name: the name of the operation resource</span>
+<span class="sd"> """</span>
+ <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">operations</span><span class="p">()</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="n">name</span><span class="p">)</span><span [...]
+ <span class="k">return</span> <span class="n">resp</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.poll_operation_until_done"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.poll_operation_until_done">[docs]</a> <span class="k">def</span> <span class="nf">poll_operation_until_done</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">polling_interval_in_seconds</span><span class= [...]
+ <span class="sd">"""</span>
+<span class="sd"> Poll backup operation state until it's completed</span>
+<span class="sd"> """</span>
+ <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_operation</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
+ <span class="n">state</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s1">'metadata'</span><span class="p">][</span><span class="s1">'common'</span><span class="p">][</span><span class="s1">'state'</span><span class="p">]</span>
+ <span class="k">if</span> <span class="n">state</span> <span class="o">==</span> <span class="s1">'PROCESSING'</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Operation is processing. Re-polling state in </span><span class="si">{}</span><span class="s1"> seconds'</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">polling_interval_in_seconds</span><span class="p">))</span>
+ <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="n">polling_interval_in_seconds</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">result</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.export_to_storage_bucket"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.export_to_storage_bucket">[docs]</a> <span class="k">def</span> <span class="nf">export_to_storage_bucket</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="n">namespace</span><span class="o">=</span><span c [...]
+ <span class="n">entity_filter</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Export entities from Cloud Datastore to Cloud Storage for backup</span>
+<span class="sd"> """</span>
+ <span class="n">output_uri_prefix</span> <span class="o">=</span> <span class="s1">'gs://'</span> <span class="o">+</span> <span class="s1">'/'</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="p">[</span><span class="n">bucket</span><span class="p">,</span> <span class="n">namespace</span><span class="p">]))</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">entity_filter</span><span class="p">:</span>
+ <span class="n">entity_filter</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">labels</span><span class="p">:</span>
+ <span class="n">labels</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="n">body</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'outputUrlPrefix'</span><span class="p">:</span> <span class="n">output_uri_prefix</span><span class="p">,</span>
+ <span class="s1">'entityFilter'</span><span class="p">:</span> <span class="n">entity_filter</span><span class="p">,</span>
+ <span class="s1">'labels'</span><span class="p">:</span> <span class="n">labels</span><span class="p">,</span>
+ <span class="p">}</span>
+ <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">admin_connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">export</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">resp</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.import_from_storage_bucket"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.import_from_storage_bucket">[docs]</a> <span class="k">def</span> <span class="nf">import_from_storage_bucket</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="n">file</span><span class="p">,</span>
+ <span class="n">namespace</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">entity_filter</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Import a backup from Cloud Storage to Cloud Datastore</span>
+<span class="sd"> """</span>
+ <span class="n">input_url</span> <span class="o">=</span> <span class="s1">'gs://'</span> <span class="o">+</span> <span class="s1">'/'</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="p">[</span><span class="n">bucket</span><span class="p">,</span> <span class="n">namespace</span><span class="p">,</span> <span cla [...]
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">entity_filter</span><span class="p">:</span>
+ <span class="n">entity_filter</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">labels</span><span class="p">:</span>
+ <span class="n">labels</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="n">body</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s1">'inputUrl'</span><span class="p">:</span> <span class="n">input_url</span><span class="p">,</span>
+ <span class="s1">'entityFilter'</span><span class="p">:</span> <span class="n">entity_filter</span><span class="p">,</span>
+ <span class="s1">'labels'</span><span class="p">:</span> <span class="n">labels</span><span class="p">,</span>
+ <span class="p">}</span>
+ <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">admin_connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">import_</span><span class="p">(</span>
+ <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">resp</span></div></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+ <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/discord_webhook_hook.html b/1.10.2/_modules/airflow/contrib/hooks/discord_webhook_hook.html
new file mode 100644
index 0000000..36aa985
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/discord_webhook_hook.html
@@ -0,0 +1,359 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.discord_webhook_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+ <div class="version">
+ 1.10.2
+ </div>
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../../index.html">Module code</a> »</li>
+
+ <li>airflow.contrib.hooks.discord_webhook_hook</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.contrib.hooks.discord_webhook_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+<span class="kn">import</span> <span class="nn">json</span>
+<span class="kn">import</span> <span class="nn">re</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.http_hook</span> <span class="k">import</span> <span class="n">HttpHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+
+
+<div class="viewcode-block" id="DiscordWebhookHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.discord_webhook_hook.DiscordWebhookHook">[docs]</a><span class="k">class</span> <span class="nc">DiscordWebhookHook</span><span class="p">(</span><span class="n">HttpHook</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> This hook allows you to post messages to Discord using incoming webhooks.</span>
+<span class="sd"> Takes a Discord connection ID with a default relative webhook endpoint. The</span>
+<span class="sd"> default endpoint can be overridden using the webhook_endpoint parameter</span>
+<span class="sd"> (https://discordapp.com/developers/docs/resources/webhook).</span>
+
+<span class="sd"> Each Discord webhook can be pre-configured to use a specific username and</span>
+<span class="sd"> avatar_url. You can override these defaults in this hook.</span>
+
+<span class="sd"> :param http_conn_id: Http connection ID with host as "https://discord.com/api/" and</span>
+<span class="sd"> default webhook endpoint in the extra field in the form of</span>
+<span class="sd"> {"webhook_endpoint": "webhooks/{webhook.id}/{webhook.token}"}</span>
+<span class="sd"> :type http_conn_id: str</span>
+<span class="sd"> :param webhook_endpoint: Discord webhook endpoint in the form of</span>
+<span class="sd"> "webhooks/{webhook.id}/{webhook.token}"</span>
+<span class="sd"> :type webhook_endpoint: str</span>
+<span class="sd"> :param message: The message you want to send to your Discord channel</span>
+<span class="sd"> (max 2000 characters)</span>
+<span class="sd"> :type message: str</span>
+<span class="sd"> :param username: Override the default username of the webhook</span>
+<span class="sd"> :type username: str</span>
+<span class="sd"> :param avatar_url: Override the default avatar of the webhook</span>
+<span class="sd"> :type avatar_url: str</span>
+<span class="sd"> :param tts: Is a text-to-speech message</span>
+<span class="sd"> :type tts: bool</span>
+<span class="sd"> :param proxy: Proxy to use to make the Discord webhook call</span>
+<span class="sd"> :type proxy: str</span>
+<span class="sd"> """</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+ <span class="n">http_conn_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">webhook_endpoint</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">message</span><span class="o">=</span><span class="s2">""</span><span class="p">,</span>
+ <span class="n">username</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">avatar_url</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">tts</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">proxy</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">DiscordWebhookHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">http_conn_id</span> <span class="o">=</span> <span class="n">http_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">webhook_endpoint</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_webhook_endpoint</span><span class="p">(</span><span class="n">http_conn_id</span><span class="p">,</span> <span class="n">webhook_endpoint</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">message</span> <span class="o">=</span> <span class="n">message</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="o">=</span> <span class="n">username</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">avatar_url</span> <span class="o">=</span> <span class="n">avatar_url</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">tts</span> <span class="o">=</span> <span class="n">tts</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">proxy</span> <span class="o">=</span> <span class="n">proxy</span>
+
+ <span class="k">def</span> <span class="nf">_get_webhook_endpoint</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">http_conn_id</span><span class="p">,</span> <span class="n">webhook_endpoint</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Given a Discord http_conn_id, return the default webhook endpoint or override if a</span>
+<span class="sd"> webhook_endpoint is manually supplied.</span>
+
+<span class="sd"> :param http_conn_id: The provided connection ID</span>
+<span class="sd"> :param webhook_endpoint: The manually provided webhook endpoint</span>
+<span class="sd"> :return: Webhook endpoint (str) to use</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="n">webhook_endpoint</span><span class="p">:</span>
+ <span class="n">endpoint</span> <span class="o">=</span> <span class="n">webhook_endpoint</span>
+ <span class="k">elif</span> <span class="n">http_conn_id</span><span class="p">:</span>
+ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">http_conn_id</span><span class="p">)</span>
+ <span class="n">extra</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span>
+ <span class="n">endpoint</span> <span class="o">=</span> <span class="n">extra</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'webhook_endpoint'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Cannot get webhook endpoint: No valid Discord '</span>
+ <span class="s1">'webhook endpoint or http_conn_id supplied.'</span><span class="p">)</span>
+
+ <span class="c1"># make sure endpoint matches the expected Discord webhook format</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="s1">'^webhooks/[0-9]+/[a-zA-Z0-9_-]+$'</span><span class="p">,</span> <span class="n">endpoint</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Expected Discord webhook endpoint in the form '</span>
+ <span class="s1">'of "webhooks/</span><span class="si">{webhook.id}</span><span class="s1">/</span><span class="si">{webhook.token}</span><span class="s1">".'</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">endpoint</span>
+
+ <span class="k">def</span> <span class="nf">_build_discord_payload</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Construct the Discord JSON payload. All relevant parameters are combined here</span>
+<span class="sd"> to a valid Discord JSON payload.</span>
+
+<span class="sd"> :return: Discord payload (str) to send</span>
+<span class="sd"> """</span>
+ <span class="n">payload</span> <span class="o">=</span> <span class="p">{}</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">:</span>
+ <span class="n">payload</span><span class="p">[</span><span class="s1">'username'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">username</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">avatar_url</span><span class="p">:</span>
+ <span class="n">payload</span><span class="p">[</span><span class="s1">'avatar_url'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">avatar_url</span>
+
+ <span class="n">payload</span><span class="p">[</span><span class="s1">'tts'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">tts</span>
+
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">message</span><span class="p">)</span> <span class="o"><=</span> <span class="mi">2000</span><span class="p">:</span>
+ <span class="n">payload</span><span class="p">[</span><span class="s1">'content'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">message</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Discord message length must be 2000 or fewer '</span>
+ <span class="s1">'characters.'</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
+
+<div class="viewcode-block" id="DiscordWebhookHook.execute"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.discord_webhook_hook.DiscordWebhookHook.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Execute the Discord webhook call</span>
+<span class="sd"> """</span>
+ <span class="n">proxies</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">proxy</span><span class="p">:</span>
+ <span class="c1"># we only need https proxy for Discord</span>
+ <span class="n">proxies</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'https'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">proxy</span><span class="p">}</span>
+
+ <span class="n">discord_payload</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_discord_payload</span><span class="p">()</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">endpoint</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">webhook_endpoint</span><span class="p">,</span>
+ <span class="n">data</span><span class="o">=</span><span class="n">discord_payload</span><span class="p">,</span>
+ <span class="n">headers</span><span class="o">=</span><span class="p">{</span><span class="s1">'Content-type'</span><span class="p">:</span> <span class="s1">'application/json'</span><span class="p">},</span>
+ <span class="n">extra_options</span><span class="o">=</span><span class="p">{</span><span class="s1">'proxies'</span><span class="p">:</span> <span class="n">proxies</span><span class="p">})</span></div></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+ <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/emr_hook.html b/1.10.2/_modules/airflow/contrib/hooks/emr_hook.html
new file mode 100644
index 0000000..366836c
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/emr_hook.html
@@ -0,0 +1,276 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.contrib.hooks.emr_hook — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+
+
+ <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+ <div class="version">
+ 1.10.2
+ </div>
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
... 155476 lines suppressed ...