You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by fo...@apache.org on 2018/01/31 11:58:40 UTC
incubator-airflow git commit: [AIRFLOW-2044] Add SparkSubmitOperator
to documentation
Repository: incubator-airflow
Updated Branches:
refs/heads/master 80d2ee8ac -> 6ee4bbd4b
[AIRFLOW-2044] Add SparkSubmitOperator to documentation
Added community contributed SparkSubmitOperator to
API documentation
Closes #2987 from Debdutto/docs/added-spark-
submit-operator
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/6ee4bbd4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/6ee4bbd4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/6ee4bbd4
Branch: refs/heads/master
Commit: 6ee4bbd4b1bc4b3f275f7946e2bcdd123970e2dd
Parents: 80d2ee8
Author: Debdutto Chakraborty <de...@gmail.com>
Authored: Wed Jan 31 12:58:34 2018 +0100
Committer: Fokko Driesprong <fo...@godatadriven.com>
Committed: Wed Jan 31 12:58:34 2018 +0100
----------------------------------------------------------------------
airflow/contrib/operators/spark_submit_operator.py | 10 +++++-----
docs/code.rst | 1 +
2 files changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/6ee4bbd4/airflow/contrib/operators/spark_submit_operator.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/spark_submit_operator.py b/airflow/contrib/operators/spark_submit_operator.py
index d743393..2eb8919 100644
--- a/airflow/contrib/operators/spark_submit_operator.py
+++ b/airflow/contrib/operators/spark_submit_operator.py
@@ -43,19 +43,19 @@ class SparkSubmitOperator(BaseOperator):
:param java_class: the main class of the Java application
:type java_class: str
:param packages: Comma-separated list of maven coordinates of jars to include on the
- driver and executor classpaths
+ driver and executor classpaths
:type packages: str
:param exclude_packages: Comma-separated list of maven coordinates of jars to exclude
- while resolving the dependencies provided in 'packages'
+ while resolving the dependencies provided in 'packages'
:type exclude_packages: str
:param repositories: Comma-separated list of additional remote repositories to search
- for the maven coordinates given with 'packages'
+ for the maven coordinates given with 'packages'
:type repositories: str
:param total_executor_cores: (Standalone & Mesos only) Total cores for all executors
- (Default: all the available cores on the worker)
+ (Default: all the available cores on the worker)
:type total_executor_cores: int
:param executor_cores: (Standalone & YARN only) Number of cores per executor
- (Default: 2)
+ (Default: 2)
:type executor_cores: int
:param executor_memory: Memory per executor (e.g. 1000M, 2G) (Default: 1G)
:type executor_memory: str
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/6ee4bbd4/docs/code.rst
----------------------------------------------------------------------
diff --git a/docs/code.rst b/docs/code.rst
index d74d00e..41a61d7 100644
--- a/docs/code.rst
+++ b/docs/code.rst
@@ -114,6 +114,7 @@ Community-contributed Operators
.. autoclass:: airflow.contrib.operators.ssh_operator.SSHOperator
.. autoclass:: airflow.contrib.operators.vertica_operator.VerticaOperator
.. autoclass:: airflow.contrib.operators.vertica_to_hive.VerticaToHiveTransfer
+.. autoclass:: airflow.contrib.operators.spark_submit_operator.SparkSubmitOperator
.. autoclass:: airflow.contrib.sensors.aws_redshift_cluster_sensor.AwsRedshiftClusterSensor
.. autoclass:: airflow.contrib.sensors.bash_sensor.BashSensor