You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2018/08/27 16:27:11 UTC
[51/51] [partial] incubator-airflow-site git commit: 1.10.0
1.10.0
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/commit/11437c14
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/tree/11437c14
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/diff/11437c14
Branch: refs/heads/asf-site
Commit: 11437c14a3607f6b85b6610dc776f55d78e7d767
Parents: 28a3eb6
Author: Kaxil Naik <ka...@gmail.com>
Authored: Mon Aug 27 17:22:22 2018 +0100
Committer: Kaxil Naik <ka...@gmail.com>
Committed: Mon Aug 27 17:22:22 2018 +0100
----------------------------------------------------------------------
_images/connection_create.png | Bin 0 -> 41547 bytes
_images/connection_edit.png | Bin 0 -> 53636 bytes
_images/connections.png | Bin 93057 -> 48442 bytes
_modules/S3_hook.html | 489 -
.../contrib/executors/mesos_executor.html | 89 +-
.../contrib/hooks/aws_dynamodb_hook.html | 300 +
_modules/airflow/contrib/hooks/aws_hook.html | 410 +
.../airflow/contrib/hooks/aws_lambda_hook.html | 302 +
.../airflow/contrib/hooks/bigquery_hook.html | 889 +-
.../airflow/contrib/hooks/databricks_hook.html | 462 +
.../airflow/contrib/hooks/datadog_hook.html | 375 +
.../airflow/contrib/hooks/datastore_hook.html | 52 +-
.../contrib/hooks/discord_webhook_hook.html | 375 +
_modules/airflow/contrib/hooks/emr_hook.html | 35 +-
_modules/airflow/contrib/hooks/fs_hook.html | 281 +
_modules/airflow/contrib/hooks/ftp_hook.html | 494 +
.../contrib/hooks/gcp_api_base_hook.html | 379 +
.../contrib/hooks/gcp_dataflow_hook.html | 196 +-
.../contrib/hooks/gcp_dataproc_hook.html | 463 +
.../contrib/hooks/gcp_mlengine_hook.html | 20 +-
.../airflow/contrib/hooks/gcp_pubsub_hook.html | 519 +
_modules/airflow/contrib/hooks/gcs_hook.html | 315 +-
.../airflow/contrib/hooks/jenkins_hook.html | 283 +
_modules/airflow/contrib/hooks/jira_hook.html | 319 +
_modules/airflow/contrib/hooks/pinot_hook.html | 340 +
_modules/airflow/contrib/hooks/qubole_hook.html | 449 +
_modules/airflow/contrib/hooks/redis_hook.html | 328 +
.../airflow/contrib/hooks/redshift_hook.html | 348 +
_modules/airflow/contrib/hooks/sftp_hook.html | 404 +
.../contrib/hooks/slack_webhook_hook.html | 364 +
.../airflow/contrib/hooks/spark_jdbc_hook.html | 481 +
.../airflow/contrib/hooks/spark_sql_hook.html | 396 +
.../contrib/hooks/spark_submit_hook.html | 799 ++
_modules/airflow/contrib/hooks/sqoop_hook.html | 580 +
_modules/airflow/contrib/hooks/ssh_hook.html | 470 +
.../airflow/contrib/hooks/vertica_hook.html | 288 +
_modules/airflow/contrib/hooks/wasb_hook.html | 94 +-
_modules/airflow/contrib/kubernetes/secret.html | 276 +
.../contrib/operators/awsbatch_operator.html | 403 +
.../operators/bigquery_check_operator.html | 37 +-
.../contrib/operators/bigquery_get_data.html | 351 +
.../contrib/operators/bigquery_operator.html | 399 +-
.../bigquery_table_delete_operator.html | 301 +
.../contrib/operators/bigquery_to_bigquery.html | 39 +-
.../contrib/operators/bigquery_to_gcs.html | 39 +-
.../contrib/operators/databricks_operator.html | 45 +-
.../contrib/operators/dataflow_operator.html | 241 +-
.../contrib/operators/dataproc_operator.html | 866 +-
.../operators/datastore_export_operator.html | 344 +
.../operators/datastore_import_operator.html | 332 +
.../operators/discord_webhook_operator.html | 333 +
.../airflow/contrib/operators/ecs_operator.html | 31 +-
.../operators/emr_add_steps_operator.html | 33 +-
.../operators/emr_create_job_flow_operator.html | 33 +-
.../emr_terminate_job_flow_operator.html | 31 +-
.../airflow/contrib/operators/file_to_gcs.html | 310 +
.../airflow/contrib/operators/file_to_wasb.html | 35 +-
.../operators/gcs_download_operator.html | 56 +-
.../contrib/operators/gcs_list_operator.html | 326 +
.../airflow/contrib/operators/gcs_operator.html | 357 +
.../airflow/contrib/operators/gcs_to_bq.html | 288 +-
.../airflow/contrib/operators/gcs_to_gcs.html | 365 +
.../airflow/contrib/operators/gcs_to_s3.html | 347 +
.../contrib/operators/hipchat_operator.html | 35 +-
.../operators/jenkins_job_trigger_operator.html | 484 +
.../contrib/operators/jira_operator.html | 329 +
.../operators/kubernetes_pod_operator.html | 362 +
.../contrib/operators/mlengine_operator.html | 260 +-
.../airflow/contrib/operators/mysql_to_gcs.html | 524 +
.../operators/postgres_to_gcs_operator.html | 481 +
.../contrib/operators/pubsub_operator.html | 669 ++
.../contrib/operators/qubole_operator.html | 411 +
.../contrib/operators/s3_list_operator.html | 317 +
.../contrib/operators/s3_to_gcs_operator.html | 425 +
.../contrib/operators/sftp_operator.html | 336 +
.../operators/slack_webhook_operator.html | 327 +
.../contrib/operators/spark_jdbc_operator.html | 449 +
.../contrib/operators/spark_sql_operator.html | 344 +
.../operators/spark_submit_operator.html | 405 +
.../contrib/operators/sqoop_operator.html | 467 +
.../airflow/contrib/operators/ssh_operator.html | 394 +
.../contrib/operators/vertica_operator.html | 284 +
.../sensors/aws_redshift_cluster_sensor.html | 287 +
.../airflow/contrib/sensors/bash_sensor.html | 331 +
.../contrib/sensors/bigquery_sensor.html | 306 +
.../airflow/contrib/sensors/datadog_sensor.html | 318 +
.../contrib/sensors/emr_base_sensor.html | 291 +
.../contrib/sensors/emr_job_flow_sensor.html | 288 +
.../contrib/sensors/emr_step_sensor.html | 292 +
.../airflow/contrib/sensors/file_sensor.html | 306 +
.../airflow/contrib/sensors/ftp_sensor.html | 302 +
.../airflow/contrib/sensors/gcs_sensor.html | 408 +
.../airflow/contrib/sensors/hdfs_sensor.html | 313 +
.../airflow/contrib/sensors/jira_sensor.html | 382 +
.../airflow/contrib/sensors/pubsub_sensor.html | 340 +
.../airflow/contrib/sensors/qubole_sensor.html | 323 +
.../contrib/sensors/redis_key_sensor.html | 282 +
.../airflow/contrib/sensors/sftp_sensor.html | 287 +
.../airflow/contrib/sensors/wasb_sensor.html | 33 +-
_modules/airflow/executors/celery_executor.html | 201 +-
_modules/airflow/executors/local_executor.html | 240 +-
.../airflow/executors/sequential_executor.html | 37 +-
_modules/airflow/hooks/S3_hook.html | 200 +-
_modules/airflow/hooks/dbapi_hook.html | 515 +
_modules/airflow/hooks/hdfs_hook.html | 335 +
_modules/airflow/hooks/http_hook.html | 358 +
_modules/airflow/hooks/mssql_hook.html | 287 +
_modules/airflow/hooks/mysql_hook.html | 341 +
_modules/airflow/hooks/oracle_hook.html | 382 +
_modules/airflow/hooks/pig_hook.html | 329 +
_modules/airflow/hooks/postgres_hook.html | 323 +
_modules/airflow/hooks/presto_hook.html | 366 +
_modules/airflow/hooks/slack_hook.html | 296 +
_modules/airflow/hooks/sqlite_hook.html | 276 +
_modules/airflow/hooks/webhdfs_hook.html | 344 +
_modules/airflow/macros.html | 31 +-
_modules/airflow/macros/hive.html | 52 +-
_modules/airflow/models.html | 1004 +-
_modules/airflow/operators/bash_operator.html | 376 +
_modules/airflow/operators/check_operator.html | 497 +
_modules/airflow/operators/dagrun_operator.html | 317 +
_modules/airflow/operators/docker_operator.html | 469 -
_modules/airflow/operators/dummy_operator.html | 273 +
_modules/airflow/operators/email_operator.html | 313 +
.../airflow/operators/generic_transfer.html | 315 +
_modules/airflow/operators/http_operator.html | 327 +
.../airflow/operators/latest_only_operator.html | 300 +
_modules/airflow/operators/mssql_operator.html | 291 +
_modules/airflow/operators/mysql_operator.html | 295 +
_modules/airflow/operators/oracle_operator.html | 290 +
_modules/airflow/operators/pig_operator.html | 307 +
.../airflow/operators/postgres_operator.html | 297 +
.../operators/presto_check_operator.html | 361 +
_modules/airflow/operators/presto_to_mysql.html | 314 +
_modules/airflow/operators/python_operator.html | 613 +
.../operators/redshift_to_s3_operator.html | 118 +-
.../operators/s3_file_transform_operator.html | 96 +-
.../airflow/operators/s3_to_hive_operator.html | 482 -
.../operators/s3_to_redshift_operator.html | 330 +
_modules/airflow/operators/sensors.html | 929 --
_modules/airflow/operators/slack_operator.html | 366 +
_modules/airflow/operators/sqlite_operator.html | 286 +
_modules/airflow/operators/subdag_operator.html | 338 +
.../airflow/sensors/base_sensor_operator.html | 310 +
.../airflow/sensors/external_task_sensor.html | 336 +
_modules/airflow/sensors/hdfs_sensor.html | 352 +
.../airflow/sensors/hive_partition_sensor.html | 310 +
_modules/airflow/sensors/http_sensor.html | 327 +
.../sensors/metastore_partition_sensor.html | 318 +
.../sensors/named_hive_partition_sensor.html | 339 +
_modules/airflow/sensors/s3_key_sensor.html | 316 +
_modules/airflow/sensors/s3_prefix_sensor.html | 303 +
_modules/airflow/sensors/sql_sensor.html | 293 +
_modules/airflow/sensors/time_delta_sensor.html | 281 +
_modules/airflow/sensors/time_sensor.html | 275 +
_modules/airflow/sensors/web_hdfs_sensor.html | 279 +
_modules/bash_operator.html | 350 -
_modules/bigquery_hook.html | 1279 --
_modules/cloudant_hook.html | 275 -
_modules/dagrun_operator.html | 310 -
_modules/dbapi_hook.html | 499 -
_modules/druid_hook.html | 325 -
_modules/dummy_operator.html | 266 -
_modules/email_operator.html | 297 -
_modules/ftp_hook.html | 486 -
_modules/gcs_hook.html | 498 -
_modules/generic_transfer.html | 308 -
_modules/hive_hooks.html | 996 --
_modules/hive_operator.html | 339 -
_modules/hive_to_druid.html | 454 -
_modules/hive_to_mysql.html | 339 -
_modules/hive_to_samba_operator.html | 260 -
_modules/http_hook.html | 351 -
_modules/http_operator.html | 320 -
_modules/index.html | 155 +-
_modules/mssql_hook.html | 242 -
_modules/mssql_operator.html | 247 -
_modules/mssql_to_hive.html | 326 -
_modules/mysql_hook.html | 334 -
_modules/mysql_operator.html | 288 -
_modules/mysql_to_hive.html | 368 -
_modules/postgres_hook.html | 316 -
_modules/postgres_operator.html | 288 -
_modules/presto_check_operator.html | 354 -
_modules/presto_hook.html | 346 -
_modules/python_operator.html | 591 -
_modules/qubole_operator.html | 404 -
_modules/s3_to_hive_operator.html | 482 -
_modules/sensors.html | 929 --
_modules/slack_operator.html | 352 -
_modules/sqlite_hook.html | 269 -
_modules/ssh_execute_operator.html | 387 -
_modules/ssh_hook.html | 386 -
_modules/vertica_hook.html | 247 -
_modules/vertica_operator.html | 246 -
_modules/vertica_to_hive.html | 330 -
_modules/webhdfs_hook.html | 300 -
_sources/api.rst.txt | 13 +-
_sources/cli.txt | 11 -
_sources/code.rst.txt | 336 +-
_sources/code.txt | 252 -
_sources/concepts.rst.txt | 49 +-
_sources/concepts.txt | 759 --
_sources/configuration.rst.txt | 310 -
_sources/configuration.txt | 230 -
_sources/faq.rst.txt | 43 +-
_sources/faq.txt | 142 -
_sources/howto/executor/use-celery.rst.txt | 47 +
_sources/howto/executor/use-dask.rst.txt | 33 +
_sources/howto/executor/use-mesos.rst.txt | 65 +
_sources/howto/index.rst.txt | 23 +
_sources/howto/initialize-database.rst.txt | 31 +
_sources/howto/manage-connections.rst.txt | 135 +
_sources/howto/run-with-systemd.rst.txt | 15 +
_sources/howto/run-with-upstart.rst.txt | 20 +
_sources/howto/secure-connections.rst.txt | 32 +
_sources/howto/set-config.rst.txt | 35 +
_sources/howto/use-test-config.rst.txt | 17 +
_sources/howto/write-logs.rst.txt | 159 +
_sources/index.rst.txt | 4 +-
_sources/index.txt | 87 -
_sources/installation.rst.txt | 29 +-
_sources/installation.txt | 90 -
_sources/integration.rst.txt | 428 +-
_sources/kubernetes.rst.txt | 101 +
_sources/license.rst.txt | 35 +-
_sources/license.txt | 211 -
_sources/lineage.rst.txt | 85 +
_sources/plugins.rst.txt | 18 +-
_sources/plugins.txt | 139 -
_sources/profiling.rst.txt | 2 +-
_sources/profiling.txt | 39 -
_sources/project.rst.txt | 7 +
_sources/project.txt | 59 -
_sources/scheduler.rst.txt | 4 +-
_sources/scheduler.txt | 101 -
_sources/security.rst.txt | 22 +-
_sources/security.txt | 253 -
_sources/start.rst.txt | 7 +-
_sources/start.txt | 49 -
_sources/timezone.rst.txt | 143 +
_sources/tutorial.rst.txt | 4 +-
_sources/tutorial.txt | 429 -
_sources/ui.txt | 102 -
_static/apache.jpg | Bin 43364 -> 0 bytes
_static/basic.css | 2 +-
_static/default.css | 3 -
_static/doctools.js | 4 +-
_static/fonts/Inconsolata.ttf | Bin 63184 -> 0 bytes
_static/incubator.jpg | Bin 91227 -> 0 bytes
_static/jquery-1.11.1.js | 10308 -----------------
_static/searchtools.js | 2 +-
_static/websupport.js | 2 +-
api.html | 18 +-
cli.html | 256 +-
code.html | 10181 ++++++++++++----
concepts.html | 50 +-
configuration.html | 520 -
faq.html | 50 +-
genindex.html | 938 +-
howto/executor/use-celery.html | 301 +
howto/executor/use-dask.html | 287 +
howto/executor/use-mesos.html | 326 +
howto/index.html | 295 +
howto/initialize-database.html | 292 +
howto/manage-connections.html | 381 +
howto/run-with-systemd.html | 274 +
howto/run-with-upstart.html | 276 +
howto/secure-connections.html | 290 +
howto/set-config.html | 289 +
howto/use-test-config.html | 275 +
howto/write-logs.html | 430 +
index.html | 163 +-
installation.html | 32 +-
integration.html | 2801 ++++-
kubernetes.html | 369 +
license.html | 381 +-
lineage.html | 328 +
objects.inv | Bin 3480 -> 6013 bytes
plugins.html | 22 +-
profiling.html | 6 +-
project.html | 11 +-
py-modindex.html | 34 +-
scheduler.html | 8 +-
search.html | 4 +-
searchindex.js | 2 +-
security.html | 41 +-
start.html | 11 +-
timezone.html | 374 +
tutorial.html | 12 +-
ui.html | 8 +-
291 files changed, 63681 insertions(+), 38174 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_images/connection_create.png
----------------------------------------------------------------------
diff --git a/_images/connection_create.png b/_images/connection_create.png
new file mode 100644
index 0000000..8a574d4
Binary files /dev/null and b/_images/connection_create.png differ
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_images/connection_edit.png
----------------------------------------------------------------------
diff --git a/_images/connection_edit.png b/_images/connection_edit.png
new file mode 100644
index 0000000..c6d14da
Binary files /dev/null and b/_images/connection_edit.png differ
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_images/connections.png
----------------------------------------------------------------------
diff --git a/_images/connections.png b/_images/connections.png
index d07a130..3a28473 100644
Binary files a/_images/connections.png and b/_images/connections.png differ
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/S3_hook.html
----------------------------------------------------------------------
diff --git a/_modules/S3_hook.html b/_modules/S3_hook.html
deleted file mode 100644
index ae1091a..0000000
--- a/_modules/S3_hook.html
+++ /dev/null
@@ -1,489 +0,0 @@
-
-
-<!DOCTYPE html>
-<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
-<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
-<head>
- <meta charset="utf-8">
-
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
-
- <title>S3_hook — Airflow Documentation</title>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../genindex.html"/>
- <link rel="search" title="Search" href="../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../index.html"/>
- <link rel="up" title="Module code" href="index.html"/>
-
-
- <script src="../_static/js/modernizr.min.js"></script>
-
-</head>
-
-<body class="wy-body-for-nav" role="document">
-
-
- <div class="wy-grid-for-nav">
-
-
- <nav data-toggle="wy-nav-shift" class="wy-nav-side">
- <div class="wy-side-scroll">
- <div class="wy-side-nav-search">
-
-
-
- <a href="../index.html" class="icon icon-home"> Airflow
-
-
-
- </a>
-
-
-
-
-
-
-
-<div role="search">
- <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
- <input type="text" name="q" placeholder="Search docs" />
- <input type="hidden" name="check_keywords" value="yes" />
- <input type="hidden" name="area" value="default" />
- </form>
-</div>
-
-
- </div>
-
- <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
-
-
-
-
-
-
- <ul>
-<li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../configuration.html">Configuration</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../ui.html">UI / Screenshots</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../concepts.html">Concepts</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../profiling.html">Data Profiling</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../cli.html">Command Line Interface</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../scheduler.html">Scheduling & Triggers</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../plugins.html">Plugins</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../api.html">Experimental Rest API</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../integration.html">Integration</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../code.html">API Reference</a></li>
-</ul>
-
-
-
- </div>
- </div>
- </nav>
-
- <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
-
-
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
-
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="../index.html">Airflow</a>
-
- </nav>
-
-
-
- <div class="wy-nav-content">
- <div class="rst-content">
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-<div role="navigation" aria-label="breadcrumbs navigation">
-
- <ul class="wy-breadcrumbs">
-
- <li><a href="../index.html">Docs</a> »</li>
-
- <li><a href="index.html">Module code</a> »</li>
-
- <li>S3_hook</li>
-
-
- <li class="wy-breadcrumbs-aside">
-
-
-
- </li>
-
- </ul>
-
-
- <hr/>
-</div>
- <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
- <div itemprop="articleBody">
-
- <h1>Source code for S3_hook</h1><div class="highlight"><pre>
-<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
-<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
-
-<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
-<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.aws_hook</span> <span class="k">import</span> <span class="n">AwsHook</span>
-
-<span class="kn">from</span> <span class="nn">six</span> <span class="k">import</span> <span class="n">BytesIO</span>
-<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="k">import</span> <span class="n">urlparse</span>
-<span class="kn">import</span> <span class="nn">re</span>
-<span class="kn">import</span> <span class="nn">fnmatch</span>
-
-<div class="viewcode-block" id="S3Hook"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook">[docs]</a><span class="k">class</span> <span class="nc">S3Hook</span><span class="p">(</span><span class="n">AwsHook</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Interact with AWS S3, using the boto3 library.</span>
-<span class="sd"> """</span>
-
- <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_client_type</span><span class="p">(</span><span class="s1">'s3'</span><span class="p">)</span>
-
- <span class="nd">@staticmethod</span>
- <span class="k">def</span> <span class="nf">parse_s3_url</span><span class="p">(</span><span class="n">s3url</span><span class="p">):</span>
- <span class="n">parsed_url</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">s3url</span><span class="p">)</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span><span class="p">:</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Please provide a bucket_name'</span><span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">bucket_name</span> <span class="o">=</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span>
- <span class="n">key</span> <span class="o">=</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">'/'</span><span class="p">)</span>
- <span class="k">return</span> <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span>
-
-<div class="viewcode-block" id="S3Hook.check_for_bucket"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.check_for_bucket">[docs]</a> <span class="k">def</span> <span class="nf">check_for_bucket</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Check if bucket_name exists.</span>
-
-<span class="sd"> :param bucket_name: the name of the bucket</span>
-<span class="sd"> :type bucket_name: str</span>
-<span class="sd"> """</span>
- <span class="k">try</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">head_bucket</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">)</span>
- <span class="k">return</span> <span class="kc">True</span>
- <span class="k">except</span><span class="p">:</span>
- <span class="k">return</span> <span class="kc">False</span></div>
-
-<div class="viewcode-block" id="S3Hook.get_bucket"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.get_bucket">[docs]</a> <span class="k">def</span> <span class="nf">get_bucket</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Returns a boto3.S3.Bucket object</span>
-
-<span class="sd"> :param bucket_name: the name of the bucket</span>
-<span class="sd"> :type bucket_name: str</span>
-<span class="sd"> """</span>
- <span class="n">s3</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_resource_type</span><span class="p">(</span><span class="s1">'s3'</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">s3</span><span class="o">.</span><span class="n">Bucket</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">)</span></div>
-
-<div class="viewcode-block" id="S3Hook.check_for_prefix"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.check_for_prefix">[docs]</a> <span class="k">def</span> <span class="nf">check_for_prefix</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="p">,</span> <span class="n">delimiter</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Checks that a prefix exists in a bucket</span>
-<span class="sd"> """</span>
- <span class="n">prefix</span> <span class="o">=</span> <span class="n">prefix</span> <span class="o">+</span> <span class="n">delimiter</span> <span class="k">if</span> <span class="n">prefix</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="o">!=</span> <span class="n">delimiter</span> <span class="k">else</span> <span class="n">prefix</span>
- <span class="n">prefix_split</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="sa">r</span><span class="s1">'(\w+[</span><span class="si">{d}</span><span class="s1">])$'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">d</span><span class="o">=</span><span class="n">delimiter</span><span class="p">),</span> <span class="n">prefix</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
- <span class="n">previous_level</span> <span class="o">=</span> <span class="n">prefix_split</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
- <span class="n">plist</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">list_prefixes</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">previous_level</span><span class="p">,</span> <span class="n">delimiter</span><span class="p">)</span>
- <span class="k">return</span> <span class="kc">False</span> <span class="k">if</span> <span class="n">plist</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">prefix</span> <span class="ow">in</span> <span class="n">plist</span></div>
-
-<div class="viewcode-block" id="S3Hook.list_prefixes"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.list_prefixes">[docs]</a> <span class="k">def</span> <span class="nf">list_prefixes</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">''</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Lists prefixes in a bucket under prefix</span>
-
-<span class="sd"> :param bucket_name: the name of the bucket</span>
-<span class="sd"> :type bucket_name: str</span>
-<span class="sd"> :param prefix: a key prefix</span>
-<span class="sd"> :type prefix: str</span>
-<span class="sd"> :param delimiter: the delimiter marks key hierarchy.</span>
-<span class="sd"> :type delimiter: str</span>
-<span class="sd"> """</span>
- <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">list_objects_v2</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span>
- <span class="n">Prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">,</span>
- <span class="n">Delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span>
- <span class="k">return</span> <span class="p">[</span><span class="n">p</span><span class="p">[</span><span class="s1">'Prefix'</span><span class="p">]</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">response</span><span class="p">[</span><span class="s1">'CommonPrefixes'</span><span class="p">]]</span> <span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'CommonPrefixes'</span><span class="p">)</span> <span class="k">else</span> <span class="kc">None</span></div>
-
-<div class="viewcode-block" id="S3Hook.list_keys"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.list_keys">[docs]</a> <span class="k">def</span> <span class="nf">list_keys</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">''</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Lists keys in a bucket under prefix and not containing delimiter</span>
-
-<span class="sd"> :param bucket_name: the name of the bucket</span>
-<span class="sd"> :type bucket_name: str</span>
-<span class="sd"> :param prefix: a key prefix</span>
-<span class="sd"> :type prefix: str</span>
-<span class="sd"> :param delimiter: the delimiter marks key hierarchy.</span>
-<span class="sd"> :type delimiter: str</span>
-<span class="sd"> """</span>
- <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">list_objects_v2</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span>
- <span class="n">Prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">,</span>
- <span class="n">Delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span>
- <span class="k">return</span> <span class="p">[</span><span class="n">k</span><span class="p">[</span><span class="s1">'Key'</span><span class="p">]</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">response</span><span class="p">[</span><span class="s1">'Contents'</span><span class="p">]]</span> <span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Contents'</span><span class="p">)</span> <span class="k">else</span> <span class="kc">None</span></div>
-
-<div class="viewcode-block" id="S3Hook.check_for_key"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.check_for_key">[docs]</a> <span class="k">def</span> <span class="nf">check_for_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Checks if a key exists in a bucket</span>
-
-<span class="sd"> :param key: S3 key that will point to the file</span>
-<span class="sd"> :type key: str</span>
-<span class="sd"> :param bucket_name: Name of the bucket in which the file is stored</span>
-<span class="sd"> :type bucket_name: str</span>
-<span class="sd"> """</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">bucket_name</span><span class="p">:</span>
- <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
-
- <span class="k">try</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">head_object</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">Key</span><span class="o">=</span><span class="n">key</span><span class="p">)</span>
- <span class="k">return</span> <span class="kc">True</span>
- <span class="k">except</span><span class="p">:</span>
- <span class="k">return</span> <span class="kc">False</span></div>
-
-<div class="viewcode-block" id="S3Hook.get_key"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.get_key">[docs]</a> <span class="k">def</span> <span class="nf">get_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Returns a boto3.s3.Object</span>
-
-<span class="sd"> :param key: the path to the key</span>
-<span class="sd"> :type key: str</span>
-<span class="sd"> :param bucket_name: the name of the bucket</span>
-<span class="sd"> :type bucket_name: str</span>
-<span class="sd"> """</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">bucket_name</span><span class="p">:</span>
- <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
-
- <span class="n">obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_resource_type</span><span class="p">(</span><span class="s1">'s3'</span><span class="p">)</span><span class="o">.</span><span class="n">Object</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span>
- <span class="n">obj</span><span class="o">.</span><span class="n">load</span><span class="p">()</span>
- <span class="k">return</span> <span class="n">obj</span></div>
-
-<div class="viewcode-block" id="S3Hook.read_key"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.read_key">[docs]</a> <span class="k">def</span> <span class="nf">read_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Reads a key from S3</span>
-
-<span class="sd"> :param key: S3 key that will point to the file</span>
-<span class="sd"> :type key: str</span>
-<span class="sd"> :param bucket_name: Name of the bucket in which the file is stored</span>
-<span class="sd"> :type bucket_name: str</span>
-<span class="sd"> """</span>
-
- <span class="n">obj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_key</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">obj</span><span class="o">.</span><span class="n">get</span><span class="p">()[</span><span class="s1">'Body'</span><span class="p">]</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span></div>
-
-<div class="viewcode-block" id="S3Hook.check_for_wildcard_key"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.check_for_wildcard_key">[docs]</a> <span class="k">def</span> <span class="nf">check_for_wildcard_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
- <span class="n">wildcard_key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">''</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Checks that a key matching a wildcard expression exists in a bucket</span>
-<span class="sd"> """</span>
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_wildcard_key</span><span class="p">(</span><span class="n">wildcard_key</span><span class="o">=</span><span class="n">wildcard_key</span><span class="p">,</span>
- <span class="n">bucket_name</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span>
- <span class="n">delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span></div>
-
-<div class="viewcode-block" id="S3Hook.get_wildcard_key"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.get_wildcard_key">[docs]</a> <span class="k">def</span> <span class="nf">get_wildcard_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">wildcard_key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">''</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Returns a boto3.s3.Object object matching the regular expression</span>
-
-<span class="sd"> :param regex_key: the path to the key</span>
-<span class="sd"> :type regex_key: str</span>
-<span class="sd"> :param bucket_name: the name of the bucket</span>
-<span class="sd"> :type bucket_name: str</span>
-<span class="sd"> """</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">bucket_name</span><span class="p">:</span>
- <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">wildcard_key</span><span class="p">)</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="n">wildcard_key</span><span class="p">)</span>
-
- <span class="n">prefix</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="sa">r</span><span class="s1">'[*]'</span><span class="p">,</span> <span class="n">wildcard_key</span><span class="p">,</span> <span class="mi">1</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
- <span class="n">klist</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">list_keys</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">)</span>
- <span class="k">if</span> <span class="n">klist</span><span class="p">:</span>
- <span class="n">key_matches</span> <span class="o">=</span> <span class="p">[</span><span class="n">k</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">klist</span> <span class="k">if</span> <span class="n">fnmatch</span><span class="o">.</span><span class="n">fnmatch</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="n">wildcard_key</span><span class="p">)]</span>
- <span class="k">if</span> <span class="n">key_matches</span><span class="p">:</span>
- <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_key</span><span class="p">(</span><span class="n">key_matches</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">bucket_name</span><span class="p">)</span></div>
-
-<div class="viewcode-block" id="S3Hook.load_file"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.load_file">[docs]</a> <span class="k">def</span> <span class="nf">load_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
- <span class="n">filename</span><span class="p">,</span>
- <span class="n">key</span><span class="p">,</span>
- <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">replace</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">encrypt</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Loads a local file to S3</span>
-
-<span class="sd"> :param filename: name of the file to load.</span>
-<span class="sd"> :type filename: str</span>
-<span class="sd"> :param key: S3 key that will point to the file</span>
-<span class="sd"> :type key: str</span>
-<span class="sd"> :param bucket_name: Name of the bucket in which to store the file</span>
-<span class="sd"> :type bucket_name: str</span>
-<span class="sd"> :param replace: A flag to decide whether or not to overwrite the key</span>
-<span class="sd"> if it already exists. If replace is False and the key exists, an</span>
-<span class="sd"> error will be raised.</span>
-<span class="sd"> :type replace: bool</span>
-<span class="sd"> :param encrypt: If True, the file will be encrypted on the server-side</span>
-<span class="sd"> by S3 and will be stored in an encrypted form while at rest in S3.</span>
-<span class="sd"> :type encrypt: bool</span>
-<span class="sd"> """</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">bucket_name</span><span class="p">:</span>
- <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
-
- <span class="k">if</span> <span class="ow">not</span> <span class="n">replace</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">check_for_key</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">):</span>
- <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"The key </span><span class="si">{key}</span><span class="s2"> already exists."</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">))</span>
-
- <span class="n">extra_args</span><span class="o">=</span><span class="p">{}</span>
- <span class="k">if</span> <span class="n">encrypt</span><span class="p">:</span>
- <span class="n">extra_args</span><span class="p">[</span><span class="s1">'ServerSideEncryption'</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"AES256"</span>
-
- <span class="n">client</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
- <span class="n">client</span><span class="o">.</span><span class="n">upload_file</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">ExtraArgs</span><span class="o">=</span><span class="n">extra_args</span><span class="p">)</span></div>
-
-<div class="viewcode-block" id="S3Hook.load_string"><a class="viewcode-back" href="../code.html#airflow.hooks.S3Hook.load_string">[docs]</a> <span class="k">def</span> <span class="nf">load_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
- <span class="n">string_data</span><span class="p">,</span>
- <span class="n">key</span><span class="p">,</span>
- <span class="n">bucket_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">replace</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">encrypt</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">encoding</span><span class="o">=</span><span class="s1">'utf-8'</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Loads a string to S3</span>
-
-<span class="sd"> This is provided as a convenience to drop a string in S3. It uses the</span>
-<span class="sd"> boto infrastructure to ship a file to s3. </span>
-
-<span class="sd"> :param string_data: string to set as content for the key.</span>
-<span class="sd"> :type string_data: str</span>
-<span class="sd"> :param key: S3 key that will point to the file</span>
-<span class="sd"> :type key: str</span>
-<span class="sd"> :param bucket_name: Name of the bucket in which to store the file</span>
-<span class="sd"> :type bucket_name: str</span>
-<span class="sd"> :param replace: A flag to decide whether or not to overwrite the key</span>
-<span class="sd"> if it already exists</span>
-<span class="sd"> :type replace: bool</span>
-<span class="sd"> :param encrypt: If True, the file will be encrypted on the server-side</span>
-<span class="sd"> by S3 and will be stored in an encrypted form while at rest in S3.</span>
-<span class="sd"> :type encrypt: bool</span>
-<span class="sd"> """</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">bucket_name</span><span class="p">:</span>
- <span class="p">(</span><span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_s3_url</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
-
- <span class="k">if</span> <span class="ow">not</span> <span class="n">replace</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">check_for_key</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">):</span>
- <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"The key </span><span class="si">{key}</span><span class="s2"> already exists."</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">))</span>
-
- <span class="n">extra_args</span><span class="o">=</span><span class="p">{}</span>
- <span class="k">if</span> <span class="n">encrypt</span><span class="p">:</span>
- <span class="n">extra_args</span><span class="p">[</span><span class="s1">'ServerSideEncryption'</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"AES256"</span>
-
- <span class="n">filelike_buffer</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">(</span><span class="n">string_data</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">encoding</span><span class="p">))</span>
-
- <span class="n">client</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
- <span class="n">client</span><span class="o">.</span><span class="n">upload_fileobj</span><span class="p">(</span><span class="n">filelike_buffer</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">ExtraArgs</span><span class="o">=</span><span class="n">extra_args</span><span class="p">)</span></div></div>
-</pre></div>
-
- </div>
- <div class="articleComments">
-
- </div>
- </div>
- <footer>
-
-
- <hr/>
-
- <div role="contentinfo">
- <p>
-
- </p>
- </div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
-
-</footer>
-
- </div>
- </div>
-
- </section>
-
- </div>
-
-
-
-
-
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../',
- VERSION:'',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../_static/jquery.js"></script>
- <script type="text/javascript" src="../_static/underscore.js"></script>
- <script type="text/javascript" src="../_static/doctools.js"></script>
-
-
-
-
-
- <script type="text/javascript" src="../_static/js/theme.js"></script>
-
-
-
-
- <script type="text/javascript">
- jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
- });
- </script>
-
-
-</body>
-</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/11437c14/_modules/airflow/contrib/executors/mesos_executor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/executors/mesos_executor.html b/_modules/airflow/contrib/executors/mesos_executor.html
index f23df55..312fffe 100644
--- a/_modules/airflow/contrib/executors/mesos_executor.html
+++ b/_modules/airflow/contrib/executors/mesos_executor.html
@@ -91,7 +91,7 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
@@ -99,8 +99,10 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -169,17 +171,22 @@
<h1>Source code for airflow.contrib.executors.mesos_executor</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
-<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
-<span class="c1"># you may not use this file except in compliance with the License.</span>
-<span class="c1"># You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c1"># See the License for the specific language governing permissions and</span>
-<span class="c1"># limitations under the License.</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1"># </span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1"># </span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
<span class="kn">from</span> <span class="nn">future</span> <span class="k">import</span> <span class="n">standard_library</span>
@@ -206,9 +213,9 @@
<span class="k">def</span> <span class="nf">get_framework_name</span><span class="p">():</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FRAMEWORK_NAME'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FRAMEWORK_NAME'</span><span class="p">):</span>
<span class="k">return</span> <span class="n">DEFAULT_FRAMEWORK_NAME</span>
- <span class="k">return</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FRAMEWORK_NAME'</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FRAMEWORK_NAME'</span><span class="p">)</span>
<span class="c1"># AirflowMesosScheduler, implements Mesos Scheduler interface</span>
@@ -232,11 +239,16 @@
<span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span> <span class="o">=</span> <span class="n">task_mem</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_counter</span> <span class="o">=</span> <span class="mi">0</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span> <span class="o">=</span> <span class="p">{}</span>
+ <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DOCKER_IMAGE_SLAVE'</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mesos_slave_docker_image</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+ <span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DOCKER_IMAGE_SLAVE'</span>
+ <span class="p">)</span>
<span class="k">def</span> <span class="nf">registered</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">frameworkId</span><span class="p">,</span> <span class="n">masterInfo</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler registered to Mesos with framework ID </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
- <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">)</span> <span class="ow">and</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">)</span> <span class="ow">and</span> \
+ <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span>
<span class="c1"># Import here to work around a circular import error</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">Connection</span>
@@ -322,6 +334,21 @@
<span class="n">command</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">cmd</span>
<span class="n">task</span><span class="o">.</span><span class="n">command</span><span class="o">.</span><span class="n">MergeFrom</span><span class="p">(</span><span class="n">command</span><span class="p">)</span>
+ <span class="c1"># If docker image for airflow is specified in config then pull that</span>
+ <span class="c1"># image before running the above airflow command</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mesos_slave_docker_image</span><span class="p">:</span>
+ <span class="n">network</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="o">.</span><span class="n">DockerInfo</span><span class="o">.</span><span class="n">Network</span><span class="o">.</span><span class="n">Value</span><span class="p">(</span><span class="s1">'BRIDGE'</span><span class="p">)</span>
+ <span class="n">docker</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="o">.</span><span class="n">DockerInfo</span><span class="p">(</span>
+ <span class="n">image</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mesos_slave_docker_image</span><span class="p">,</span>
+ <span class="n">force_pull_image</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">network</span><span class="o">=</span><span class="n">network</span>
+ <span class="p">)</span>
+ <span class="n">container</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="p">(</span>
+ <span class="nb">type</span><span class="o">=</span><span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="o">.</span><span class="n">DOCKER</span><span class="p">,</span>
+ <span class="n">docker</span><span class="o">=</span><span class="n">docker</span>
+ <span class="p">)</span>
+ <span class="n">task</span><span class="o">.</span><span class="n">container</span><span class="o">.</span><span class="n">MergeFrom</span><span class="p">(</span><span class="n">container</span><span class="p">)</span>
+
<span class="n">tasks</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">task</span><span class="p">)</span>
<span class="n">remainingCpus</span> <span class="o">-=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span>
@@ -371,28 +398,28 @@
<span class="n">framework</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">FrameworkInfo</span><span class="p">()</span>
<span class="n">framework</span><span class="o">.</span><span class="n">user</span> <span class="o">=</span> <span class="s1">''</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'MASTER'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'MASTER'</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Expecting mesos master URL for mesos executor"</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"mesos.master not provided for mesos executor"</span><span class="p">)</span>
- <span class="n">master</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'MASTER'</span><span class="p">)</span>
+ <span class="n">master</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'MASTER'</span><span class="p">)</span>
<span class="n">framework</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">get_framework_name</span><span class="p">()</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_CPU'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_CPU'</span><span class="p">):</span>
<span class="n">task_cpu</span> <span class="o">=</span> <span class="mi">1</span>
<span class="k">else</span><span class="p">:</span>
- <span class="n">task_cpu</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_CPU'</span><span class="p">)</span>
+ <span class="n">task_cpu</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_CPU'</span><span class="p">)</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_MEMORY'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_MEMORY'</span><span class="p">):</span>
<span class="n">task_memory</span> <span class="o">=</span> <span class="mi">256</span>
<span class="k">else</span><span class="p">:</span>
- <span class="n">task_memory</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_MEMORY'</span><span class="p">)</span>
+ <span class="n">task_memory</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_MEMORY'</span><span class="p">)</span>
- <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">):</span>
<span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="kc">True</span>
- <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span>
<span class="c1"># Import here to work around a circular import error</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">Connection</span>
@@ -404,7 +431,9 @@
<span class="c1"># Set the Framework ID to let the scheduler reconnect with running tasks.</span>
<span class="n">framework</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">connection</span><span class="o">.</span><span class="n">extra</span>
- <span class="n">framework</span><span class="o">.</span><span class="n">failover_timeout</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">)</span>
+ <span class="n">framework</span><span class="o">.</span><span class="n">failover_timeout</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span>
+ <span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span>
+ <span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="kc">False</span>
@@ -415,17 +444,17 @@
<span class="n">implicit_acknowledgements</span> <span class="o">=</span> <span class="mi">1</span>
- <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'AUTHENTICATE'</span><span class="p">):</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_PRINCIPAL'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'AUTHENTICATE'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_PRINCIPAL'</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Expecting authentication principal in the environment"</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"mesos.default_principal not provided in authenticated mode"</span><span class="p">)</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_SECRET'</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_SECRET'</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Expecting authentication secret in the environment"</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"mesos.default_secret not provided in authenticated mode"</span><span class="p">)</span>
<span class="n">credential</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">Credential</span><span class="p">()</span>
- <span class="n">credential</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_PRINCIPAL'</span><span class="p">)</span>
- <span class="n">credential</span><span class="o">.</span><span class="n">secret</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_SECRET'</span><span class="p">)</span>
+ <span class="n">credential</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_PRINCIPAL'</span><span class="p">)</span>
+ <span class="n">credential</span><span class="o">.</span><span class="n">secret</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_SECRET'</span><span class="p">)</span>
<span class="n">framework</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="n">credential</span><span class="o">.</span><span class="n">principal</span>
@@ -446,7 +475,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span> <span class="o">=</span> <span class="n">driver</span>
<span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
- <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">executor_config</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>