You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by as...@apache.org on 2018/11/29 15:58:36 UTC
[38/38] incubator-airflow-site git commit: Docs from 1.10.1
Docs from 1.10.1
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/commit/1f06fa0e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/tree/1f06fa0e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/diff/1f06fa0e
Branch: refs/heads/asf-site
Commit: 1f06fa0e0ead70d24f3c87326734aec89930207d
Parents: 7d4d762
Author: Ash Berlin-Taylor <as...@firemirror.com>
Authored: Thu Nov 29 15:57:25 2018 +0000
Committer: Ash Berlin-Taylor <as...@firemirror.com>
Committed: Thu Nov 29 15:57:25 2018 +0000
----------------------------------------------------------------------
_images/airflow.gif | Bin 622963 -> 622832 bytes
.../contrib/executors/mesos_executor.html | 548 ---
.../contrib/hooks/aws_dynamodb_hook.html | 21 +-
.../contrib/hooks/aws_firehose_hook.html | 267 ++
_modules/airflow/contrib/hooks/aws_hook.html | 104 +-
.../airflow/contrib/hooks/aws_lambda_hook.html | 21 +-
.../contrib/hooks/azure_data_lake_hook.html | 21 +-
.../contrib/hooks/azure_fileshare_hook.html | 21 +-
.../airflow/contrib/hooks/bigquery_hook.html | 139 +-
.../airflow/contrib/hooks/cassandra_hook.html | 21 +-
.../airflow/contrib/hooks/cloudant_hook.html | 21 +-
.../airflow/contrib/hooks/databricks_hook.html | 87 +-
.../airflow/contrib/hooks/datadog_hook.html | 21 +-
.../airflow/contrib/hooks/datastore_hook.html | 21 +-
.../contrib/hooks/discord_webhook_hook.html | 21 +-
_modules/airflow/contrib/hooks/emr_hook.html | 36 +-
_modules/airflow/contrib/hooks/fs_hook.html | 21 +-
_modules/airflow/contrib/hooks/ftp_hook.html | 91 +-
.../contrib/hooks/gcp_api_base_hook.html | 23 +-
.../contrib/hooks/gcp_container_hook.html | 25 +-
.../contrib/hooks/gcp_dataflow_hook.html | 128 +-
.../contrib/hooks/gcp_dataproc_hook.html | 21 +-
.../contrib/hooks/gcp_function_hook.html | 406 +++
.../contrib/hooks/gcp_mlengine_hook.html | 32 +-
.../airflow/contrib/hooks/gcp_pubsub_hook.html | 21 +-
.../airflow/contrib/hooks/gcp_sql_hook.html | 474 +++
_modules/airflow/contrib/hooks/gcs_hook.html | 60 +-
_modules/airflow/contrib/hooks/imap_hook.html | 491 +++
.../airflow/contrib/hooks/jenkins_hook.html | 21 +-
_modules/airflow/contrib/hooks/jira_hook.html | 21 +-
_modules/airflow/contrib/hooks/mongo_hook.html | 21 +-
_modules/airflow/contrib/hooks/pinot_hook.html | 21 +-
_modules/airflow/contrib/hooks/qubole_hook.html | 21 +-
_modules/airflow/contrib/hooks/redis_hook.html | 21 +-
.../airflow/contrib/hooks/redshift_hook.html | 21 +-
.../airflow/contrib/hooks/sagemaker_hook.html | 972 ++++++
.../airflow/contrib/hooks/segment_hook.html | 21 +-
_modules/airflow/contrib/hooks/sftp_hook.html | 98 +-
.../contrib/hooks/slack_webhook_hook.html | 21 +-
.../airflow/contrib/hooks/snowflake_hook.html | 21 +-
.../airflow/contrib/hooks/spark_jdbc_hook.html | 21 +-
.../airflow/contrib/hooks/spark_sql_hook.html | 21 +-
.../contrib/hooks/spark_submit_hook.html | 21 +-
_modules/airflow/contrib/hooks/sqoop_hook.html | 21 +-
_modules/airflow/contrib/hooks/ssh_hook.html | 315 +-
.../airflow/contrib/hooks/vertica_hook.html | 21 +-
_modules/airflow/contrib/hooks/wasb_hook.html | 64 +-
_modules/airflow/contrib/hooks/winrm_hook.html | 21 +-
_modules/airflow/contrib/kubernetes/secret.html | 21 +-
.../contrib/operators/awsbatch_operator.html | 47 +-
.../operators/bigquery_check_operator.html | 71 +-
.../contrib/operators/bigquery_get_data.html | 25 +-
.../contrib/operators/bigquery_operator.html | 202 +-
.../bigquery_table_delete_operator.html | 25 +-
.../contrib/operators/bigquery_to_bigquery.html | 33 +-
.../contrib/operators/bigquery_to_gcs.html | 34 +-
.../contrib/operators/cassandra_to_gcs.html | 25 +-
.../contrib/operators/databricks_operator.html | 37 +-
.../contrib/operators/dataflow_operator.html | 104 +-
.../contrib/operators/dataproc_operator.html | 130 +-
.../operators/datastore_export_operator.html | 25 +-
.../operators/datastore_import_operator.html | 25 +-
.../operators/discord_webhook_operator.html | 21 +-
.../contrib/operators/druid_operator.html | 25 +-
.../airflow/contrib/operators/ecs_operator.html | 52 +-
.../operators/emr_add_steps_operator.html | 25 +-
.../operators/emr_create_job_flow_operator.html | 25 +-
.../emr_terminate_job_flow_operator.html | 25 +-
.../airflow/contrib/operators/file_to_gcs.html | 34 +-
.../airflow/contrib/operators/file_to_wasb.html | 21 +-
.../contrib/operators/gcp_compute_operator.html | 394 +++
.../operators/gcp_container_operator.html | 29 +-
.../operators/gcp_function_operator.html | 522 +++
.../contrib/operators/gcp_sql_operator.html | 737 ++++
.../operators/gcs_download_operator.html | 25 +-
.../contrib/operators/gcs_list_operator.html | 25 +-
.../airflow/contrib/operators/gcs_operator.html | 25 +-
.../airflow/contrib/operators/gcs_to_bq.html | 25 +-
.../airflow/contrib/operators/gcs_to_gcs.html | 25 +-
.../airflow/contrib/operators/gcs_to_s3.html | 25 +-
.../contrib/operators/hipchat_operator.html | 25 +-
.../contrib/operators/hive_to_dynamodb.html | 25 +-
.../operators/jenkins_job_trigger_operator.html | 25 +-
.../contrib/operators/jira_operator.html | 25 +-
.../operators/kubernetes_pod_operator.html | 51 +-
.../contrib/operators/mlengine_operator.html | 41 +-
.../airflow/contrib/operators/mongo_to_s3.html | 21 +-
.../airflow/contrib/operators/mysql_to_gcs.html | 25 +-
.../operators/postgres_to_gcs_operator.html | 25 +-
.../contrib/operators/pubsub_operator.html | 41 +-
.../operators/qubole_check_operator.html | 29 +-
.../contrib/operators/qubole_operator.html | 29 +-
.../contrib/operators/s3_list_operator.html | 25 +-
.../contrib/operators/s3_to_gcs_operator.html | 25 +-
.../operators/sagemaker_base_operator.html | 311 ++
.../sagemaker_endpoint_config_operator.html | 278 ++
.../operators/sagemaker_endpoint_operator.html | 362 ++
.../operators/sagemaker_model_operator.html | 279 ++
.../operators/sagemaker_training_operator.html | 310 ++
.../operators/sagemaker_transform_operator.html | 336 ++
.../operators/sagemaker_tuning_operator.html | 310 ++
.../operators/segment_track_event_operator.html | 25 +-
.../contrib/operators/sftp_operator.html | 57 +-
.../operators/slack_webhook_operator.html | 25 +-
.../contrib/operators/snowflake_operator.html | 25 +-
.../contrib/operators/spark_jdbc_operator.html | 25 +-
.../contrib/operators/spark_sql_operator.html | 25 +-
.../operators/spark_submit_operator.html | 25 +-
.../contrib/operators/sqoop_operator.html | 25 +-
.../airflow/contrib/operators/ssh_operator.html | 164 +-
.../contrib/operators/vertica_operator.html | 25 +-
.../contrib/operators/vertica_to_hive.html | 25 +-
.../contrib/operators/winrm_operator.html | 25 +-
.../sensors/aws_redshift_cluster_sensor.html | 21 +-
.../airflow/contrib/sensors/bash_sensor.html | 21 +-
.../contrib/sensors/bigquery_sensor.html | 21 +-
.../contrib/sensors/cassandra_sensor.html | 271 ++
.../airflow/contrib/sensors/datadog_sensor.html | 21 +-
.../contrib/sensors/emr_base_sensor.html | 21 +-
.../contrib/sensors/emr_job_flow_sensor.html | 21 +-
.../contrib/sensors/emr_step_sensor.html | 25 +-
.../airflow/contrib/sensors/file_sensor.html | 21 +-
.../airflow/contrib/sensors/ftp_sensor.html | 21 +-
.../airflow/contrib/sensors/gcs_sensor.html | 21 +-
.../airflow/contrib/sensors/hdfs_sensor.html | 21 +-
.../airflow/contrib/sensors/jira_sensor.html | 21 +-
.../airflow/contrib/sensors/pubsub_sensor.html | 21 +-
.../airflow/contrib/sensors/qubole_sensor.html | 21 +-
.../contrib/sensors/redis_key_sensor.html | 21 +-
.../contrib/sensors/sagemaker_base_sensor.html | 285 ++
.../sensors/sagemaker_endpoint_sensor.html | 272 ++
.../sensors/sagemaker_training_sensor.html | 313 ++
.../sensors/sagemaker_transform_sensor.html | 273 ++
.../sensors/sagemaker_tuning_sensor.html | 273 ++
.../airflow/contrib/sensors/sftp_sensor.html | 21 +-
.../airflow/contrib/sensors/wasb_sensor.html | 21 +-
_modules/airflow/executors/celery_executor.html | 27 +-
_modules/airflow/executors/local_executor.html | 21 +-
.../airflow/executors/sequential_executor.html | 21 +-
_modules/airflow/hooks/S3_hook.html | 164 +-
_modules/airflow/hooks/dbapi_hook.html | 21 +-
_modules/airflow/hooks/docker_hook.html | 21 +-
_modules/airflow/hooks/druid_hook.html | 35 +-
_modules/airflow/hooks/hdfs_hook.html | 21 +-
_modules/airflow/hooks/hive_hooks.html | 21 +-
_modules/airflow/hooks/http_hook.html | 33 +-
_modules/airflow/hooks/jdbc_hook.html | 21 +-
_modules/airflow/hooks/mssql_hook.html | 21 +-
_modules/airflow/hooks/mysql_hook.html | 33 +-
_modules/airflow/hooks/oracle_hook.html | 21 +-
_modules/airflow/hooks/pig_hook.html | 21 +-
_modules/airflow/hooks/postgres_hook.html | 21 +-
_modules/airflow/hooks/presto_hook.html | 21 +-
_modules/airflow/hooks/samba_hook.html | 21 +-
_modules/airflow/hooks/slack_hook.html | 21 +-
_modules/airflow/hooks/sqlite_hook.html | 21 +-
_modules/airflow/hooks/webhdfs_hook.html | 21 +-
_modules/airflow/hooks/zendesk_hook.html | 21 +-
_modules/airflow/macros.html | 21 +-
_modules/airflow/macros/hive.html | 21 +-
_modules/airflow/models.html | 242 +-
_modules/airflow/operators/bash_operator.html | 25 +-
_modules/airflow/operators/check_operator.html | 33 +-
_modules/airflow/operators/dagrun_operator.html | 25 +-
_modules/airflow/operators/docker_operator.html | 80 +-
.../airflow/operators/druid_check_operator.html | 25 +-
_modules/airflow/operators/dummy_operator.html | 25 +-
_modules/airflow/operators/email_operator.html | 25 +-
.../airflow/operators/generic_transfer.html | 25 +-
_modules/airflow/operators/hive_operator.html | 33 +-
.../airflow/operators/hive_stats_operator.html | 25 +-
_modules/airflow/operators/hive_to_druid.html | 25 +-
_modules/airflow/operators/hive_to_mysql.html | 25 +-
.../operators/hive_to_samba_operator.html | 25 +-
_modules/airflow/operators/http_operator.html | 36 +-
_modules/airflow/operators/jdbc_operator.html | 25 +-
.../airflow/operators/latest_only_operator.html | 25 +-
_modules/airflow/operators/mssql_operator.html | 25 +-
_modules/airflow/operators/mssql_to_hive.html | 25 +-
_modules/airflow/operators/mysql_operator.html | 25 +-
_modules/airflow/operators/mysql_to_hive.html | 25 +-
_modules/airflow/operators/oracle_operator.html | 25 +-
_modules/airflow/operators/pig_operator.html | 33 +-
.../airflow/operators/postgres_operator.html | 25 +-
.../operators/presto_check_operator.html | 21 +-
_modules/airflow/operators/presto_to_mysql.html | 25 +-
_modules/airflow/operators/python_operator.html | 33 +-
.../operators/redshift_to_s3_operator.html | 25 +-
.../operators/s3_file_transform_operator.html | 49 +-
.../airflow/operators/s3_to_hive_operator.html | 27 +-
.../operators/s3_to_redshift_operator.html | 25 +-
_modules/airflow/operators/slack_operator.html | 21 +-
_modules/airflow/operators/sqlite_operator.html | 25 +-
_modules/airflow/operators/subdag_operator.html | 25 +-
.../airflow/sensors/base_sensor_operator.html | 25 +-
.../airflow/sensors/external_task_sensor.html | 21 +-
_modules/airflow/sensors/hdfs_sensor.html | 21 +-
.../airflow/sensors/hive_partition_sensor.html | 21 +-
_modules/airflow/sensors/http_sensor.html | 21 +-
.../sensors/metastore_partition_sensor.html | 21 +-
.../sensors/named_hive_partition_sensor.html | 21 +-
_modules/airflow/sensors/s3_key_sensor.html | 21 +-
_modules/airflow/sensors/s3_prefix_sensor.html | 23 +-
_modules/airflow/sensors/sql_sensor.html | 21 +-
_modules/airflow/sensors/time_delta_sensor.html | 21 +-
_modules/airflow/sensors/time_sensor.html | 21 +-
_modules/airflow/sensors/web_hdfs_sensor.html | 21 +-
_modules/index.html | 45 +-
_sources/api.rst.txt | 17 +
_sources/cli.rst.txt | 17 +
_sources/code.rst.txt | 40 +-
_sources/concepts.rst.txt | 63 +-
_sources/faq.rst.txt | 17 +
_sources/howto/executor/use-celery.rst.txt | 19 +-
_sources/howto/executor/use-dask.rst.txt | 17 +
_sources/howto/executor/use-mesos.rst.txt | 17 +
_sources/howto/index.rst.txt | 17 +
_sources/howto/initialize-database.rst.txt | 17 +
_sources/howto/manage-connections.rst.txt | 101 +-
_sources/howto/operator.rst.txt | 493 +++
_sources/howto/run-with-systemd.rst.txt | 17 +
_sources/howto/run-with-upstart.rst.txt | 17 +
_sources/howto/secure-connections.rst.txt | 28 +-
_sources/howto/set-config.rst.txt | 37 +-
_sources/howto/use-test-config.rst.txt | 17 +
_sources/howto/write-logs.rst.txt | 86 +-
_sources/index.rst.txt | 17 +
_sources/installation.rst.txt | 84 +-
_sources/integration.rst.txt | 161 +-
_sources/kubernetes.rst.txt | 34 +-
_sources/license.rst.txt | 17 +
_sources/lineage.rst.txt | 17 +
_sources/plugins.rst.txt | 70 +-
_sources/profiling.rst.txt | 19 +
_sources/project.rst.txt | 18 +-
_sources/scheduler.rst.txt | 24 +-
_sources/security.rst.txt | 37 +-
_sources/start.rst.txt | 17 +
_sources/timezone.rst.txt | 84 +-
_sources/tutorial.rst.txt | 58 +-
_sources/ui.rst.txt | 19 +-
_static/basic.css | 11 +
_static/css/theme.css | 6 +-
_static/doctools.js | 6 +-
_static/documentation_options.js | 291 +-
_static/fonts/Inconsolata-Bold.ttf | Bin 0 -> 109948 bytes
_static/fonts/Inconsolata-Regular.ttf | Bin 0 -> 96964 bytes
_static/fonts/Inconsolata.ttf | Bin 0 -> 63184 bytes
_static/fonts/Lato-Bold.ttf | Bin 0 -> 656544 bytes
_static/fonts/Lato-Regular.ttf | Bin 0 -> 656568 bytes
_static/fonts/RobotoSlab-Bold.ttf | Bin 0 -> 170616 bytes
_static/fonts/RobotoSlab-Regular.ttf | Bin 0 -> 169064 bytes
_static/js/theme.js | 6 +-
_static/searchtools.js | 351 +-
api.html | 21 +-
cli.html | 1473 ++++----
code.html | 3218 +++++++++++++++---
concepts.html | 59 +-
faq.html | 21 +-
genindex.html | 545 ++-
howto/executor/use-celery.html | 23 +-
howto/executor/use-dask.html | 21 +-
howto/executor/use-mesos.html | 21 +-
howto/index.html | 21 +-
howto/initialize-database.html | 21 +-
howto/manage-connections.html | 100 +-
howto/operator.html | 643 +++-
howto/run-with-systemd.html | 21 +-
howto/run-with-upstart.html | 21 +-
howto/secure-connections.html | 31 +-
howto/set-config.html | 40 +-
howto/use-test-config.html | 21 +-
howto/write-logs.html | 92 +-
index.html | 137 +-
installation.html | 124 +-
integration.html | 1798 +++++++++-
kubernetes.html | 58 +-
license.html | 21 +-
lineage.html | 21 +-
objects.inv | Bin 7505 -> 8915 bytes
plugins.html | 68 +-
profiling.html | 21 +-
project.html | 22 +-
py-modindex.html | 21 +-
scheduler.html | 27 +-
search.html | 23 +-
searchindex.js | 2 +-
security.html | 41 +-
start.html | 21 +-
timezone.html | 25 +-
tutorial.html | 62 +-
ui.html | 23 +-
292 files changed, 19787 insertions(+), 6652 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_images/airflow.gif
----------------------------------------------------------------------
diff --git a/_images/airflow.gif b/_images/airflow.gif
index 1889b86..7646e37 100644
Binary files a/_images/airflow.gif and b/_images/airflow.gif differ
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/executors/mesos_executor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/executors/mesos_executor.html b/_modules/airflow/contrib/executors/mesos_executor.html
deleted file mode 100644
index a495421..0000000
--- a/_modules/airflow/contrib/executors/mesos_executor.html
+++ /dev/null
@@ -1,548 +0,0 @@
-
-
-<!DOCTYPE html>
-<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
-<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
-<head>
- <meta charset="utf-8">
-
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
-
- <title>airflow.contrib.executors.mesos_executor — Airflow Documentation</title>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
- <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
- <link rel="index" title="Index" href="../../../../genindex.html" />
- <link rel="search" title="Search" href="../../../../search.html" />
-
-
- <script src="../../../../_static/js/modernizr.min.js"></script>
-
-</head>
-
-<body class="wy-body-for-nav">
-
-
- <div class="wy-grid-for-nav">
-
-
- <nav data-toggle="wy-nav-shift" class="wy-nav-side">
- <div class="wy-side-scroll">
- <div class="wy-side-nav-search">
-
-
-
- <a href="../../../../index.html" class="icon icon-home"> Airflow
-
-
-
- </a>
-
-
-
-
-
-
-
-<div role="search">
- <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
- <input type="text" name="q" placeholder="Search docs" />
- <input type="hidden" name="check_keywords" value="yes" />
- <input type="hidden" name="area" value="default" />
- </form>
-</div>
-
-
- </div>
-
- <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
-
-
-
-
-
-
- <ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
-</ul>
-
-
-
- </div>
- </div>
- </nav>
-
- <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
-
-
- <nav class="wy-nav-top" aria-label="top navigation">
-
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="../../../../index.html">Airflow</a>
-
- </nav>
-
-
- <div class="wy-nav-content">
-
- <div class="rst-content">
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-<div role="navigation" aria-label="breadcrumbs navigation">
-
- <ul class="wy-breadcrumbs">
-
- <li><a href="../../../../index.html">Docs</a> »</li>
-
- <li><a href="../../../index.html">Module code</a> »</li>
-
- <li>airflow.contrib.executors.mesos_executor</li>
-
-
- <li class="wy-breadcrumbs-aside">
-
- </li>
-
- </ul>
-
-
- <hr/>
-</div>
- <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
- <div itemprop="articleBody">
-
- <h1>Source code for airflow.contrib.executors.mesos_executor</h1><div class="highlight"><pre>
-<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
-<span class="c1">#</span>
-<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
-<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
-<span class="c1"># distributed with this work for additional information</span>
-<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
-<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
-<span class="c1"># "License"); you may not use this file except in compliance</span>
-<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1">#</span>
-<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1">#</span>
-<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
-<span class="c1"># software distributed under the License is distributed on an</span>
-<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
-<span class="c1"># KIND, either express or implied. See the License for the</span>
-<span class="c1"># specific language governing permissions and limitations</span>
-<span class="c1"># under the License.</span>
-
-<span class="kn">from</span> <span class="nn">future</span> <span class="k">import</span> <span class="n">standard_library</span>
-
-<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
-<span class="kn">from</span> <span class="nn">airflow.www.utils</span> <span class="k">import</span> <span class="n">LoginMixin</span>
-
-
-<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
-<span class="kn">from</span> <span class="nn">queue</span> <span class="k">import</span> <span class="n">Queue</span>
-
-<span class="kn">import</span> <span class="nn">mesos.interface</span>
-<span class="kn">from</span> <span class="nn">mesos.interface</span> <span class="k">import</span> <span class="n">mesos_pb2</span>
-<span class="kn">import</span> <span class="nn">mesos.native</span>
-
-<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">configuration</span>
-<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="k">import</span> <span class="n">BaseExecutor</span>
-<span class="kn">from</span> <span class="nn">airflow.settings</span> <span class="k">import</span> <span class="n">Session</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span>
-<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
-
-<span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
-<span class="n">DEFAULT_FRAMEWORK_NAME</span> <span class="o">=</span> <span class="s1">'Airflow'</span>
-<span class="n">FRAMEWORK_CONNID_PREFIX</span> <span class="o">=</span> <span class="s1">'mesos_framework_'</span>
-
-
-<span class="k">def</span> <span class="nf">get_framework_name</span><span class="p">():</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FRAMEWORK_NAME'</span><span class="p">):</span>
- <span class="k">return</span> <span class="n">DEFAULT_FRAMEWORK_NAME</span>
- <span class="k">return</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FRAMEWORK_NAME'</span><span class="p">)</span>
-
-
-<span class="c1"># AirflowMesosScheduler, implements Mesos Scheduler interface</span>
-<span class="c1"># To schedule airflow jobs on mesos</span>
-<span class="k">class</span> <span class="nc">AirflowMesosScheduler</span><span class="p">(</span><span class="n">mesos</span><span class="o">.</span><span class="n">interface</span><span class="o">.</span><span class="n">Scheduler</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> Airflow Mesos scheduler implements mesos scheduler interface</span>
-<span class="sd"> to schedule airflow tasks on mesos.</span>
-<span class="sd"> Basically, it schedules a command like</span>
-<span class="sd"> 'airflow run <dag_id> <task_instance_id> <start_date> --local -p=<pickle>'</span>
-<span class="sd"> to run on a mesos slave.</span>
-<span class="sd"> """</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
- <span class="n">task_queue</span><span class="p">,</span>
- <span class="n">result_queue</span><span class="p">,</span>
- <span class="n">task_cpu</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
- <span class="n">task_mem</span><span class="o">=</span><span class="mi">256</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span> <span class="o">=</span> <span class="n">task_queue</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="n">result_queue</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span> <span class="o">=</span> <span class="n">task_cpu</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span> <span class="o">=</span> <span class="n">task_mem</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">task_counter</span> <span class="o">=</span> <span class="mi">0</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span> <span class="o">=</span> <span class="p">{}</span>
- <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DOCKER_IMAGE_SLAVE'</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">mesos_slave_docker_image</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
- <span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DOCKER_IMAGE_SLAVE'</span>
- <span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">registered</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">frameworkId</span><span class="p">,</span> <span class="n">masterInfo</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler registered to Mesos with framework ID </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
- <span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
-
- <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">)</span> <span class="ow">and</span> \
- <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span>
- <span class="c1"># Import here to work around a circular import error</span>
- <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">Connection</span>
-
- <span class="c1"># Update the Framework ID in the database.</span>
- <span class="n">session</span> <span class="o">=</span> <span class="n">Session</span><span class="p">()</span>
- <span class="n">conn_id</span> <span class="o">=</span> <span class="n">FRAMEWORK_CONNID_PREFIX</span> <span class="o">+</span> <span class="n">get_framework_name</span><span class="p">()</span>
- <span class="n">connection</span> <span class="o">=</span> <span class="n">Session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">Connection</span><span class="p">)</span><span class="o">.</span><span class="n">filter_by</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
- <span class="k">if</span> <span class="n">connection</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
- <span class="n">connection</span> <span class="o">=</span> <span class="n">Connection</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">,</span> <span class="n">conn_type</span><span class="o">=</span><span class="s1">'mesos_framework-id'</span><span class="p">,</span>
- <span class="n">extra</span><span class="o">=</span><span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">connection</span><span class="o">.</span><span class="n">extra</span> <span class="o">=</span> <span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span>
-
- <span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">connection</span><span class="p">)</span>
- <span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
- <span class="n">Session</span><span class="o">.</span><span class="n">remove</span><span class="p">()</span>
-
- <span class="k">def</span> <span class="nf">reregistered</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">masterInfo</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler re-registered to mesos"</span><span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">disconnected</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler disconnected from mesos"</span><span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">offerRescinded</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">offerId</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler offer </span><span class="si">%s</span><span class="s2"> rescinded"</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">offerId</span><span class="p">))</span>
-
- <span class="k">def</span> <span class="nf">frameworkMessage</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">executorId</span><span class="p">,</span> <span class="n">slaveId</span><span class="p">,</span> <span class="n">message</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"AirflowScheduler received framework message </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">executorLost</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">executorId</span><span class="p">,</span> <span class="n">slaveId</span><span class="p">,</span> <span class="n">status</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"AirflowScheduler executor </span><span class="si">%s</span><span class="s2"> lost"</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">executorId</span><span class="p">))</span>
-
- <span class="k">def</span> <span class="nf">slaveLost</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">slaveId</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"AirflowScheduler slave </span><span class="si">%s</span><span class="s2"> lost"</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">slaveId</span><span class="p">))</span>
-
- <span class="k">def</span> <span class="nf">error</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">message</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"AirflowScheduler driver aborted </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"AirflowScheduler driver aborted </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">message</span><span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">resourceOffers</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">offers</span><span class="p">):</span>
- <span class="k">for</span> <span class="n">offer</span> <span class="ow">in</span> <span class="n">offers</span><span class="p">:</span>
- <span class="n">tasks</span> <span class="o">=</span> <span class="p">[]</span>
- <span class="n">offerCpus</span> <span class="o">=</span> <span class="mi">0</span>
- <span class="n">offerMem</span> <span class="o">=</span> <span class="mi">0</span>
- <span class="k">for</span> <span class="n">resource</span> <span class="ow">in</span> <span class="n">offer</span><span class="o">.</span><span class="n">resources</span><span class="p">:</span>
- <span class="k">if</span> <span class="n">resource</span><span class="o">.</span><span class="n">name</span> <span class="o">==</span> <span class="s2">"cpus"</span><span class="p">:</span>
- <span class="n">offerCpus</span> <span class="o">+=</span> <span class="n">resource</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span>
- <span class="k">elif</span> <span class="n">resource</span><span class="o">.</span><span class="n">name</span> <span class="o">==</span> <span class="s2">"mem"</span><span class="p">:</span>
- <span class="n">offerMem</span> <span class="o">+=</span> <span class="n">resource</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span>
-
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Received offer </span><span class="si">%s</span><span class="s2"> with cpus: </span><span class="si">%s</span><span class="s2"> and mem: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
- <span class="n">offer</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">value</span><span class="p">,</span> <span class="n">offerCpus</span><span class="p">,</span> <span class="n">offerMem</span><span class="p">)</span>
-
- <span class="n">remainingCpus</span> <span class="o">=</span> <span class="n">offerCpus</span>
- <span class="n">remainingMem</span> <span class="o">=</span> <span class="n">offerMem</span>
-
- <span class="k">while</span> <span class="p">(</span><span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">empty</span><span class="p">())</span> <span class="ow">and</span> \
- <span class="n">remainingCpus</span> <span class="o">>=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span> <span class="ow">and</span> \
- <span class="n">remainingMem</span> <span class="o">>=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span><span class="p">:</span>
- <span class="n">key</span><span class="p">,</span> <span class="n">cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
- <span class="n">tid</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_counter</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">task_counter</span> <span class="o">+=</span> <span class="mi">1</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span><span class="p">[</span><span class="nb">str</span><span class="p">(</span><span class="n">tid</span><span class="p">)]</span> <span class="o">=</span> <span class="n">key</span>
-
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Launching task </span><span class="si">%d</span><span class="s2"> using offer </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">tid</span><span class="p">,</span> <span class="n">offer</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
-
- <span class="n">task</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TaskInfo</span><span class="p">()</span>
- <span class="n">task</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">tid</span><span class="p">)</span>
- <span class="n">task</span><span class="o">.</span><span class="n">slave_id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">offer</span><span class="o">.</span><span class="n">slave_id</span><span class="o">.</span><span class="n">value</span>
- <span class="n">task</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="s2">"AirflowTask </span><span class="si">%d</span><span class="s2">"</span> <span class="o">%</span> <span class="n">tid</span>
-
- <span class="n">cpus</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">add</span><span class="p">()</span>
- <span class="n">cpus</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="s2">"cpus"</span>
- <span class="n">cpus</span><span class="o">.</span><span class="n">type</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">Value</span><span class="o">.</span><span class="n">SCALAR</span>
- <span class="n">cpus</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span>
-
- <span class="n">mem</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">add</span><span class="p">()</span>
- <span class="n">mem</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="s2">"mem"</span>
- <span class="n">mem</span><span class="o">.</span><span class="n">type</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">Value</span><span class="o">.</span><span class="n">SCALAR</span>
- <span class="n">mem</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span>
-
- <span class="n">command</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">CommandInfo</span><span class="p">()</span>
- <span class="n">command</span><span class="o">.</span><span class="n">shell</span> <span class="o">=</span> <span class="kc">True</span>
- <span class="n">command</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">cmd</span>
- <span class="n">task</span><span class="o">.</span><span class="n">command</span><span class="o">.</span><span class="n">MergeFrom</span><span class="p">(</span><span class="n">command</span><span class="p">)</span>
-
- <span class="c1"># If docker image for airflow is specified in config then pull that</span>
- <span class="c1"># image before running the above airflow command</span>
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mesos_slave_docker_image</span><span class="p">:</span>
- <span class="n">network</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="o">.</span><span class="n">DockerInfo</span><span class="o">.</span><span class="n">Network</span><span class="o">.</span><span class="n">Value</span><span class="p">(</span><span class="s1">'BRIDGE'</span><span class="p">)</span>
- <span class="n">docker</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="o">.</span><span class="n">DockerInfo</span><span class="p">(</span>
- <span class="n">image</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mesos_slave_docker_image</span><span class="p">,</span>
- <span class="n">force_pull_image</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">network</span><span class="o">=</span><span class="n">network</span>
- <span class="p">)</span>
- <span class="n">container</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="p">(</span>
- <span class="nb">type</span><span class="o">=</span><span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="o">.</span><span class="n">DOCKER</span><span class="p">,</span>
- <span class="n">docker</span><span class="o">=</span><span class="n">docker</span>
- <span class="p">)</span>
- <span class="n">task</span><span class="o">.</span><span class="n">container</span><span class="o">.</span><span class="n">MergeFrom</span><span class="p">(</span><span class="n">container</span><span class="p">)</span>
-
- <span class="n">tasks</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">task</span><span class="p">)</span>
-
- <span class="n">remainingCpus</span> <span class="o">-=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span>
- <span class="n">remainingMem</span> <span class="o">-=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span>
-
- <span class="n">driver</span><span class="o">.</span><span class="n">launchTasks</span><span class="p">(</span><span class="n">offer</span><span class="o">.</span><span class="n">id</span><span class="p">,</span> <span class="n">tasks</span><span class="p">)</span>
-
- <span class="k">def</span> <span class="nf">statusUpdate</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">update</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
- <span class="s2">"Task </span><span class="si">%s</span><span class="s2"> is in state </span><span class="si">%s</span><span class="s2">, data </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
- <span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">,</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TaskState</span><span class="o">.</span><span class="n">Name</span><span class="p">(</span><span class="n">update</span><span class="o">.</span><span class="n">state</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">update</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
- <span class="p">)</span>
-
- <span class="k">try</span><span class="p">:</span>
- <span class="n">key</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span><span class="p">[</span><span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">]</span>
- <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
- <span class="c1"># The map may not contain an item if the framework re-registered</span>
- <span class="c1"># after a failover.</span>
- <span class="c1"># Discard these tasks.</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Unrecognised task key </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
- <span class="k">return</span>
-
- <span class="k">if</span> <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_FINISHED</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">))</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">task_done</span><span class="p">()</span>
-
- <span class="k">if</span> <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_LOST</span> <span class="ow">or</span> \
- <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_KILLED</span> <span class="ow">or</span> \
- <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_FAILED</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">FAILED</span><span class="p">))</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">task_done</span><span class="p">()</span>
-
-
-<div class="viewcode-block" id="MesosExecutor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor">[docs]</a><span class="k">class</span> <span class="nc">MesosExecutor</span><span class="p">(</span><span class="n">BaseExecutor</span><span class="p">,</span> <span class="n">LoginMixin</span><span class="p">):</span>
- <span class="sd">"""</span>
-<span class="sd"> MesosExecutor allows distributing the execution of task</span>
-<span class="sd"> instances to multiple mesos workers.</span>
-
-<span class="sd"> Apache Mesos is a distributed systems kernel which abstracts</span>
-<span class="sd"> CPU, memory, storage, and other compute resources away from</span>
-<span class="sd"> machines (physical or virtual), enabling fault-tolerant and</span>
-<span class="sd"> elastic distributed systems to easily be built and run effectively.</span>
-<span class="sd"> See http://mesos.apache.org/</span>
-<span class="sd"> """</span>
-<div class="viewcode-block" id="MesosExecutor.start"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor.start">[docs]</a> <span class="k">def</span> <span class="nf">start</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span> <span class="o">=</span> <span class="n">Queue</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="n">Queue</span><span class="p">()</span>
- <span class="n">framework</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">FrameworkInfo</span><span class="p">()</span>
- <span class="n">framework</span><span class="o">.</span><span class="n">user</span> <span class="o">=</span> <span class="s1">''</span>
-
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'MASTER'</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Expecting mesos master URL for mesos executor"</span><span class="p">)</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"mesos.master not provided for mesos executor"</span><span class="p">)</span>
-
- <span class="n">master</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'MASTER'</span><span class="p">)</span>
-
- <span class="n">framework</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">get_framework_name</span><span class="p">()</span>
-
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_CPU'</span><span class="p">):</span>
- <span class="n">task_cpu</span> <span class="o">=</span> <span class="mi">1</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">task_cpu</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_CPU'</span><span class="p">)</span>
-
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_MEMORY'</span><span class="p">):</span>
- <span class="n">task_memory</span> <span class="o">=</span> <span class="mi">256</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">task_memory</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_MEMORY'</span><span class="p">)</span>
-
- <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">):</span>
- <span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="kc">True</span>
-
- <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span>
- <span class="c1"># Import here to work around a circular import error</span>
- <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">Connection</span>
-
- <span class="c1"># Query the database to get the ID of the Mesos Framework, if available.</span>
- <span class="n">conn_id</span> <span class="o">=</span> <span class="n">FRAMEWORK_CONNID_PREFIX</span> <span class="o">+</span> <span class="n">framework</span><span class="o">.</span><span class="n">name</span>
- <span class="n">session</span> <span class="o">=</span> <span class="n">Session</span><span class="p">()</span>
- <span class="n">connection</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">Connection</span><span class="p">)</span><span class="o">.</span><span class="n">filter_by</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
- <span class="k">if</span> <span class="n">connection</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
- <span class="c1"># Set the Framework ID to let the scheduler reconnect</span>
- <span class="c1"># with running tasks.</span>
- <span class="n">framework</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">connection</span><span class="o">.</span><span class="n">extra</span>
-
- <span class="n">framework</span><span class="o">.</span><span class="n">failover_timeout</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span>
- <span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span>
- <span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="kc">False</span>
-
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
- <span class="s1">'MesosFramework master : </span><span class="si">%s</span><span class="s1">, name : </span><span class="si">%s</span><span class="s1">, cpu : </span><span class="si">%s</span><span class="s1">, mem : </span><span class="si">%s</span><span class="s1">, checkpoint : </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
- <span class="n">master</span><span class="p">,</span> <span class="n">framework</span><span class="o">.</span><span class="n">name</span><span class="p">,</span>
- <span class="nb">str</span><span class="p">(</span><span class="n">task_cpu</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">task_memory</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span><span class="p">)</span>
- <span class="p">)</span>
-
- <span class="n">implicit_acknowledgements</span> <span class="o">=</span> <span class="mi">1</span>
-
- <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'AUTHENTICATE'</span><span class="p">):</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_PRINCIPAL'</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Expecting authentication principal in the environment"</span><span class="p">)</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
- <span class="s2">"mesos.default_principal not provided in authenticated mode"</span><span class="p">)</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_SECRET'</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Expecting authentication secret in the environment"</span><span class="p">)</span>
- <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
- <span class="s2">"mesos.default_secret not provided in authenticated mode"</span><span class="p">)</span>
-
- <span class="n">credential</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">Credential</span><span class="p">()</span>
- <span class="n">credential</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_PRINCIPAL'</span><span class="p">)</span>
- <span class="n">credential</span><span class="o">.</span><span class="n">secret</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'DEFAULT_SECRET'</span><span class="p">)</span>
-
- <span class="n">framework</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="n">credential</span><span class="o">.</span><span class="n">principal</span>
-
- <span class="n">driver</span> <span class="o">=</span> <span class="n">mesos</span><span class="o">.</span><span class="n">native</span><span class="o">.</span><span class="n">MesosSchedulerDriver</span><span class="p">(</span>
- <span class="n">AirflowMesosScheduler</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="p">,</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="p">,</span>
- <span class="n">task_cpu</span><span class="p">,</span>
- <span class="n">task_memory</span><span class="p">),</span>
- <span class="n">framework</span><span class="p">,</span>
- <span class="n">master</span><span class="p">,</span>
- <span class="n">implicit_acknowledgements</span><span class="p">,</span>
- <span class="n">credential</span><span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="n">framework</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="s1">'Airflow'</span>
- <span class="n">driver</span> <span class="o">=</span> <span class="n">mesos</span><span class="o">.</span><span class="n">native</span><span class="o">.</span><span class="n">MesosSchedulerDriver</span><span class="p">(</span>
- <span class="n">AirflowMesosScheduler</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="p">,</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="p">,</span>
- <span class="n">task_cpu</span><span class="p">,</span>
- <span class="n">task_memory</span><span class="p">),</span>
- <span class="n">framework</span><span class="p">,</span>
- <span class="n">master</span><span class="p">,</span>
- <span class="n">implicit_acknowledgements</span><span class="p">)</span>
-
- <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span> <span class="o">=</span> <span class="n">driver</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span><span class="o">.</span><span class="n">start</span><span class="p">()</span></div>
-
-<div class="viewcode-block" id="MesosExecutor.execute_async"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor.execute_async">[docs]</a> <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">executor_config</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">))</span></div>
-
-<div class="viewcode-block" id="MesosExecutor.sync"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor.sync">[docs]</a> <span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">empty</span><span class="p">():</span>
- <span class="n">results</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">change_state</span><span class="p">(</span><span class="o">*</span><span class="n">results</span><span class="p">)</span></div>
-
-<div class="viewcode-block" id="MesosExecutor.end"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor.end">[docs]</a> <span class="k">def</span> <span class="nf">end</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span><span class="o">.</span><span class="n">stop</span><span class="p">()</span></div></div>
-</pre></div>
-
- </div>
-
- </div>
- <footer>
-
-
- <hr/>
-
- <div role="contentinfo">
- <p>
-
- </p>
- </div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
-
-</footer>
-
- </div>
- </div>
-
- </section>
-
- </div>
-
-
-
-
-
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
-
-
-
- <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
-
- <script type="text/javascript">
- jQuery(function () {
- SphinxRtdTheme.Navigation.enable(true);
- });
- </script>
-
-</body>
-</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html b/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html
index b8fea44..4feb169 100644
--- a/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html
+++ b/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html
@@ -258,20 +258,13 @@
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT:'../../../../',
- VERSION:'',
- LANGUAGE:'None',
- COLLAPSE_INDEX:false,
- FILE_SUFFIX:'.html',
- HAS_SOURCE: true,
- SOURCELINK_SUFFIX: '.txt'
- };
- </script>
- <script type="text/javascript" src="../../../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+