You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/11/13 08:40:15 UTC
[airflow] branch master updated: Docs installation improvements
(#12304)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 7825e8f Docs installation improvements (#12304)
7825e8f is described below
commit 7825e8f59034645ab3247229be83a3aa90baece1
Author: Kamil BreguĊa <mi...@users.noreply.github.com>
AuthorDate: Fri Nov 13 09:38:54 2020 +0100
Docs installation improvements (#12304)
* Improvements for installation docs
---
.pre-commit-config.yaml | 8 +-
BREEZE.rst | 2 +-
STATIC_CODE_CHECKS.rst | 2 +-
airflow/providers/amazon/provider.yaml | 24 ++
airflow/providers/apache/cassandra/provider.yaml | 24 ++
airflow/providers/apache/druid/provider.yaml | 24 ++
airflow/providers/apache/hdfs/provider.yaml | 25 ++
airflow/providers/apache/hive/provider.yaml | 24 ++
airflow/providers/apache/kylin/provider.yaml | 24 ++
airflow/providers/apache/livy/provider.yaml | 24 ++
airflow/providers/apache/pig/provider.yaml | 24 ++
airflow/providers/apache/pinot/provider.yaml | 24 ++
airflow/providers/apache/spark/provider.yaml | 24 ++
airflow/providers/apache/sqoop/provider.yaml | 24 ++
airflow/providers/celery/provider.yaml | 24 ++
airflow/providers/cloudant/provider.yaml | 24 ++
airflow/providers/cncf/kubernetes/provider.yaml | 24 ++
airflow/providers/databricks/provider.yaml | 24 ++
airflow/providers/datadog/provider.yaml | 24 ++
airflow/providers/dingding/provider.yaml | 24 ++
airflow/providers/discord/provider.yaml | 24 ++
airflow/providers/docker/provider.yaml | 24 ++
airflow/providers/elasticsearch/provider.yaml | 24 ++
airflow/providers/exasol/provider.yaml | 24 ++
airflow/providers/facebook/provider.yaml | 24 ++
airflow/providers/ftp/provider.yaml | 24 ++
airflow/providers/google/provider.yaml | 30 +++
airflow/providers/grpc/provider.yaml | 24 ++
airflow/providers/hashicorp/provider.yaml | 24 ++
airflow/providers/http/provider.yaml | 24 ++
airflow/providers/imap/provider.yaml | 24 ++
airflow/providers/jdbc/provider.yaml | 24 ++
airflow/providers/jenkins/provider.yaml | 24 ++
airflow/providers/jira/provider.yaml | 24 ++
airflow/providers/microsoft/azure/provider.yaml | 24 ++
airflow/providers/microsoft/mssql/provider.yaml | 24 ++
airflow/providers/microsoft/winrm/provider.yaml | 24 ++
airflow/providers/mongo/provider.yaml | 24 ++
airflow/providers/mysql/provider.yaml | 24 ++
airflow/providers/odbc/provider.yaml | 24 ++
airflow/providers/openfaas/provider.yaml | 24 ++
airflow/providers/opsgenie/provider.yaml | 24 ++
airflow/providers/oracle/provider.yaml | 24 ++
airflow/providers/pagerduty/provider.yaml | 24 ++
airflow/providers/papermill/provider.yaml | 24 ++
airflow/providers/plexus/provider.yaml | 24 ++
airflow/providers/postgres/provider.yaml | 24 ++
airflow/providers/presto/provider.yaml | 24 ++
airflow/providers/qubole/provider.yaml | 24 ++
airflow/providers/redis/provider.yaml | 24 ++
airflow/providers/salesforce/provider.yaml | 24 ++
airflow/providers/samba/provider.yaml | 24 ++
airflow/providers/segment/provider.yaml | 24 ++
airflow/providers/sendgrid/provider.yaml | 24 ++
airflow/providers/sftp/provider.yaml | 24 ++
airflow/providers/singularity/provider.yaml | 24 ++
airflow/providers/slack/provider.yaml | 24 ++
airflow/providers/snowflake/provider.yaml | 24 ++
airflow/providers/sqlite/provider.yaml | 24 ++
airflow/providers/ssh/provider.yaml | 24 ++
airflow/providers/vertica/provider.yaml | 24 ++
airflow/providers/yandex/provider.yaml | 24 ++
airflow/providers/zendesk/provider.yaml | 24 ++
breeze-complete | 2 +-
dev/provider.yaml.schema.json | 31 +++
docs/autoapi_templates/index.rst | 71 +++++
docs/build_docs.py | 9 +-
docs/conf.py | 44 +---
docs/{installation.rst => extra-packages-ref.rst} | 228 +---------------
docs/exts/providers_packages_ref.py | 71 +++++
docs/exts/sphinx_script_update.py | 19 +-
docs/index.rst | 3 +
docs/installation.rst | 293 ++-------------------
docs/operators-and-hooks-ref.rst | 2 +-
docs/provider-packages-ref.rst | 60 +++++
docs/provider-packages.rst | 120 +++++++++
...> pre_commit_check_setup_extra_packages_ref.py} | 2 +-
setup.py | 4 +
78 files changed, 1859 insertions(+), 559 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 1287d56..f885594 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -244,12 +244,12 @@ repos:
files: ^setup.py$
pass_filenames: false
entry: ./scripts/ci/pre_commit/pre_commit_check_order_setup.py
- - id: setup-installation
- name: Checks if all the libraries in setup.py are listed in installation.rst file
+ - id: setup-extra-packages
+ name: Checks if all the libraries in setup.py are listed in extra-packages-ref.rst file
language: python
- files: ^setup.py$|^docs/installation.rst$
+ files: ^setup.py$|^docs/extra-packages-ref.rst$
pass_filenames: false
- entry: ./scripts/ci/pre_commit/pre_commit_check_setup_installation.py
+ entry: ./scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py
additional_dependencies: ['rich==9.2.0']
- id: update-breeze-file
name: Update output of breeze command in BREEZE.rst
diff --git a/BREEZE.rst b/BREEZE.rst
index 663fd49..8313b47 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -2017,7 +2017,7 @@ This is the current syntax for `./breeze <./breeze>`_:
no-providers-in-core-examples no-relative-imports pre-commit-descriptions
provide-create-sessions providers-init-file pydevd pydocstyle pylint pylint-tests
python-no-log-warn pyupgrade restrict-start_date rst-backticks setup-order
- setup-installation shellcheck sort-in-the-wild stylelint trailing-whitespace
+ setup-extra-packages shellcheck sort-in-the-wild stylelint trailing-whitespace
update-breeze-file update-extras update-local-yml-file update-setup-cfg-file
version-sync yamllint
diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst
index a69b1d9..9ad5a53 100644
--- a/STATIC_CODE_CHECKS.rst
+++ b/STATIC_CODE_CHECKS.rst
@@ -148,7 +148,7 @@ require Breeze Docker images to be installed locally:
----------------------------------- ---------------------------------------------------------------- ------------
``setup-order`` Checks for an order of dependencies in setup.py
----------------------------------- ---------------------------------------------------------------- ------------
-``setup-installation`` Checks if all the libraries in setup.py are listed in docs
+``setup-extra-packages`` Checks if all the libraries in setup.py are listed in docs
----------------------------------- ---------------------------------------------------------------- ------------
``shellcheck`` Checks shell files with shellcheck.
----------------------------------- ---------------------------------------------------------------- ------------
diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml
new file mode 100644
index 0000000..bbb656f
--- /dev/null
+++ b/airflow/providers/amazon/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-amazon
+description: |
+ Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/apache/cassandra/provider.yaml b/airflow/providers/apache/cassandra/provider.yaml
new file mode 100644
index 0000000..d229584
--- /dev/null
+++ b/airflow/providers/apache/cassandra/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-apache-cassandra
+description: |
+ `Apache Cassandra <http://cassandra.apache.org/>`__.
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/apache/druid/provider.yaml b/airflow/providers/apache/druid/provider.yaml
new file mode 100644
index 0000000..dd760b4
--- /dev/null
+++ b/airflow/providers/apache/druid/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-apache-druid
+description: |
+ `Apache Cassandra <https://druid.apache.org/>`__.
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/apache/hdfs/provider.yaml b/airflow/providers/apache/hdfs/provider.yaml
new file mode 100644
index 0000000..4cd6b09
--- /dev/null
+++ b/airflow/providers/apache/hdfs/provider.yaml
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-apache-hdfs
+description: |
+ `Hadoop Distributed File System (HDFS) <https://hadoop.apache.org/docs/r1.2.1/hdfs_design.html>`__
+ and `WebHDFS <https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/WebHDFS.html>`__.
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/apache/hive/provider.yaml b/airflow/providers/apache/hive/provider.yaml
new file mode 100644
index 0000000..a3480d6
--- /dev/null
+++ b/airflow/providers/apache/hive/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-apache-hive
+description: |
+ `Apache Hive <https://hive.apache.org/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/apache/kylin/provider.yaml b/airflow/providers/apache/kylin/provider.yaml
new file mode 100644
index 0000000..d3800fb
--- /dev/null
+++ b/airflow/providers/apache/kylin/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-apache-kylin
+description: |
+ `Apache Kylin <https://kylin.apache.org/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/apache/livy/provider.yaml b/airflow/providers/apache/livy/provider.yaml
new file mode 100644
index 0000000..bbbb30b
--- /dev/null
+++ b/airflow/providers/apache/livy/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-apache-livy
+description: |
+ `Apache Livy <https://livy.apache.org/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/apache/pig/provider.yaml b/airflow/providers/apache/pig/provider.yaml
new file mode 100644
index 0000000..933e07c
--- /dev/null
+++ b/airflow/providers/apache/pig/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-apache-pig
+description: |
+ `Apache Pig <https://pig.apache.org/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/apache/pinot/provider.yaml b/airflow/providers/apache/pinot/provider.yaml
new file mode 100644
index 0000000..a835962
--- /dev/null
+++ b/airflow/providers/apache/pinot/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-apache-pinot
+description: |
+ `Apache Pinot <https://pinot.apache.org/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/apache/spark/provider.yaml b/airflow/providers/apache/spark/provider.yaml
new file mode 100644
index 0000000..a258281
--- /dev/null
+++ b/airflow/providers/apache/spark/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-apache-spark
+description: |
+ `Apache Spark <https://spark.apache.org/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/apache/sqoop/provider.yaml b/airflow/providers/apache/sqoop/provider.yaml
new file mode 100644
index 0000000..b7a12ed
--- /dev/null
+++ b/airflow/providers/apache/sqoop/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-apache-sqoop
+description: |
+ `Apache Sqoop <https://sqoop.apache.org/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/celery/provider.yaml b/airflow/providers/celery/provider.yaml
new file mode 100644
index 0000000..a8dbcdd
--- /dev/null
+++ b/airflow/providers/celery/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-celery
+description: |
+ `Celery <http://www.celeryproject.org/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/cloudant/provider.yaml b/airflow/providers/cloudant/provider.yaml
new file mode 100644
index 0000000..fc7c6ed
--- /dev/null
+++ b/airflow/providers/cloudant/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-cloudant
+description: |
+ `IBM Cloudant <https://www.ibm.com/cloud/cloudant>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/cncf/kubernetes/provider.yaml b/airflow/providers/cncf/kubernetes/provider.yaml
new file mode 100644
index 0000000..5c016c3
--- /dev/null
+++ b/airflow/providers/cncf/kubernetes/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-cncf-kubernetes
+description: |
+ `Kubernetes <https://kubernetes.io/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/databricks/provider.yaml b/airflow/providers/databricks/provider.yaml
new file mode 100644
index 0000000..e2c04b5
--- /dev/null
+++ b/airflow/providers/databricks/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-databricks
+description: |
+ `Databricks <https://databricks.com/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/datadog/provider.yaml b/airflow/providers/datadog/provider.yaml
new file mode 100644
index 0000000..4be6d32
--- /dev/null
+++ b/airflow/providers/datadog/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-datadog
+description: |
+ `Datadog <https://www.datadoghq.com/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/dingding/provider.yaml b/airflow/providers/dingding/provider.yaml
new file mode 100644
index 0000000..cd7882f
--- /dev/null
+++ b/airflow/providers/dingding/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-dingding
+description: |
+ `Datadog <https://oapi.dingtalk.com/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/discord/provider.yaml b/airflow/providers/discord/provider.yaml
new file mode 100644
index 0000000..a5b18e2
--- /dev/null
+++ b/airflow/providers/discord/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-discord
+description: |
+ `Discord <https://discordapp.com/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/docker/provider.yaml b/airflow/providers/docker/provider.yaml
new file mode 100644
index 0000000..3e8798f
--- /dev/null
+++ b/airflow/providers/docker/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-docker
+description: |
+ `Docker <https://docs.docker.com/install/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/elasticsearch/provider.yaml b/airflow/providers/elasticsearch/provider.yaml
new file mode 100644
index 0000000..6b07b8f
--- /dev/null
+++ b/airflow/providers/elasticsearch/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-elasticsearch
+description: |
+ `Elasticsearch <https://https//www.elastic.co/elasticsearch>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/exasol/provider.yaml b/airflow/providers/exasol/provider.yaml
new file mode 100644
index 0000000..852556e
--- /dev/null
+++ b/airflow/providers/exasol/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-exasol
+description: |
+ `Exasol <https://docs.exasol.com/home.htm>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/facebook/provider.yaml b/airflow/providers/facebook/provider.yaml
new file mode 100644
index 0000000..f194190
--- /dev/null
+++ b/airflow/providers/facebook/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-facebook
+description: |
+ `Facebook Ads <http://business.facebook.com/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/ftp/provider.yaml b/airflow/providers/ftp/provider.yaml
new file mode 100644
index 0000000..e31e823
--- /dev/null
+++ b/airflow/providers/ftp/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-ftp
+description: |
+ `File Transfer Protocol (FTP) <https://tools.ietf.org/html/rfc114>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml
new file mode 100644
index 0000000..c335aa4
--- /dev/null
+++ b/airflow/providers/google/provider.yaml
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-google
+description: |
+ Google services including:
+
+ - `Google Ads <https://ads.google.com/>`__
+ - `Google Cloud (GCP) <https://cloud.google.com/>`__
+ - `Google Firebase <https://firebase.google.com/>`__
+ - `Google Marketing Platform <https://marketingplatform.google.com/>`__
+ - `Google Workspace <https://workspace.google.pl/>`__ (formerly Google Suite)
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/grpc/provider.yaml b/airflow/providers/grpc/provider.yaml
new file mode 100644
index 0000000..866f0c6
--- /dev/null
+++ b/airflow/providers/grpc/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-grpc
+description: |
+ `gRPC <https://grpc.io/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/hashicorp/provider.yaml b/airflow/providers/hashicorp/provider.yaml
new file mode 100644
index 0000000..6fc0820
--- /dev/null
+++ b/airflow/providers/hashicorp/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-hashicorp
+description: |
+ Hashicorp including `Hashicorp Vault <https://www.vaultproject.io/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/http/provider.yaml b/airflow/providers/http/provider.yaml
new file mode 100644
index 0000000..488afeb
--- /dev/null
+++ b/airflow/providers/http/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-http
+description: |
+ `Hypertext Transfer Protocol (HTTP) <https://www.w3.org/Protocols/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/imap/provider.yaml b/airflow/providers/imap/provider.yaml
new file mode 100644
index 0000000..bcfcf0e
--- /dev/null
+++ b/airflow/providers/imap/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-imap
+description: |
+ `Internet Message Access Protocol (IMAP) <https://tools.ietf.org/html/rfc3501>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/jdbc/provider.yaml b/airflow/providers/jdbc/provider.yaml
new file mode 100644
index 0000000..a821d35
--- /dev/null
+++ b/airflow/providers/jdbc/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-jdbc
+description: |
+ `Java Database Connectivity (JDBC) <https://docs.oracle.com/javase/8/docs/technotes/guides/jdbc/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/jenkins/provider.yaml b/airflow/providers/jenkins/provider.yaml
new file mode 100644
index 0000000..5131dc7
--- /dev/null
+++ b/airflow/providers/jenkins/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-jenkins
+description: |
+ `Jenkins <https://jenkins.io/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/jira/provider.yaml b/airflow/providers/jira/provider.yaml
new file mode 100644
index 0000000..ffcd2cf
--- /dev/null
+++ b/airflow/providers/jira/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-jira
+description: |
+ `Atlassian Jira <https://www.atlassian.com/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/microsoft/azure/provider.yaml b/airflow/providers/microsoft/azure/provider.yaml
new file mode 100644
index 0000000..3d4d033
--- /dev/null
+++ b/airflow/providers/microsoft/azure/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-microsoft-azure
+description: |
+ `Microsoft Azure <https://azure.microsoft.com/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/microsoft/mssql/provider.yaml b/airflow/providers/microsoft/mssql/provider.yaml
new file mode 100644
index 0000000..d3e1b35
--- /dev/null
+++ b/airflow/providers/microsoft/mssql/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-microsoft-mssql
+description: |
+ `Microsoft SQL Server (MSSQL) <https://www.microsoft.com/en-us/sql-server/sql-server-downloads>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/microsoft/winrm/provider.yaml b/airflow/providers/microsoft/winrm/provider.yaml
new file mode 100644
index 0000000..e672488
--- /dev/null
+++ b/airflow/providers/microsoft/winrm/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-microsoft-winrm
+description: |
+ `Windows Remote Management (WinRM) <https://docs.microsoft.com/en-us/windows/win32/winrm/portal>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/mongo/provider.yaml b/airflow/providers/mongo/provider.yaml
new file mode 100644
index 0000000..14e0de7
--- /dev/null
+++ b/airflow/providers/mongo/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-mongo
+description: |
+ `MongoDB <https://www.mongodb.com/what-is-mongodb>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/mysql/provider.yaml b/airflow/providers/mysql/provider.yaml
new file mode 100644
index 0000000..10a7d3e
--- /dev/null
+++ b/airflow/providers/mysql/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-mysql
+description: |
+ `MySQL <https://www.mysql.com/products/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/odbc/provider.yaml b/airflow/providers/odbc/provider.yaml
new file mode 100644
index 0000000..5b664d8
--- /dev/null
+++ b/airflow/providers/odbc/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-odbc
+description: |
+ `ODBC <https://github.com/mkleehammer/pyodbc/wiki>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/openfaas/provider.yaml b/airflow/providers/openfaas/provider.yaml
new file mode 100644
index 0000000..65c9bda
--- /dev/null
+++ b/airflow/providers/openfaas/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-openfaas
+description: |
+ `OpenFaaS <https://www.openfaas.com/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/opsgenie/provider.yaml b/airflow/providers/opsgenie/provider.yaml
new file mode 100644
index 0000000..f75e78a
--- /dev/null
+++ b/airflow/providers/opsgenie/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-opsgenie
+description: |
+ `Opsgenie <https://www.opsgenie.com/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/oracle/provider.yaml b/airflow/providers/oracle/provider.yaml
new file mode 100644
index 0000000..a5ff2dc
--- /dev/null
+++ b/airflow/providers/oracle/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-oracle
+description: |
+ `Oracle <https://www.oracle.com/en/database/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/pagerduty/provider.yaml b/airflow/providers/pagerduty/provider.yaml
new file mode 100644
index 0000000..baaac3e
--- /dev/null
+++ b/airflow/providers/pagerduty/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-pagerduty
+description: |
+ `Pagerduty <https://www.pagerduty.com/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/papermill/provider.yaml b/airflow/providers/papermill/provider.yaml
new file mode 100644
index 0000000..1c72244
--- /dev/null
+++ b/airflow/providers/papermill/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-papermill
+description: |
+ `Papermill <https://github.com/nteract/papermill>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/plexus/provider.yaml b/airflow/providers/plexus/provider.yaml
new file mode 100644
index 0000000..d55edeb
--- /dev/null
+++ b/airflow/providers/plexus/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-plexus
+description: |
+ `Plexus <https://plexus.corescientific.com/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/postgres/provider.yaml b/airflow/providers/postgres/provider.yaml
new file mode 100644
index 0000000..ba0147a
--- /dev/null
+++ b/airflow/providers/postgres/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-postgres
+description: |
+ `PostgreSQL <https://www.postgresql.org/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/presto/provider.yaml b/airflow/providers/presto/provider.yaml
new file mode 100644
index 0000000..c197780
--- /dev/null
+++ b/airflow/providers/presto/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-presto
+description: |
+ `Presto <https://prestodb.github.io/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/qubole/provider.yaml b/airflow/providers/qubole/provider.yaml
new file mode 100644
index 0000000..dba4e0e
--- /dev/null
+++ b/airflow/providers/qubole/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-qubole
+description: |
+ `Qubole <https://www.qubole.com/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/redis/provider.yaml b/airflow/providers/redis/provider.yaml
new file mode 100644
index 0000000..290dcf7
--- /dev/null
+++ b/airflow/providers/redis/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-redis
+description: |
+ `Redis <https://redis.io/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/salesforce/provider.yaml b/airflow/providers/salesforce/provider.yaml
new file mode 100644
index 0000000..30dbc3c
--- /dev/null
+++ b/airflow/providers/salesforce/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-salesforce
+description: |
+ `Salesforce <https://www.salesforce.com/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/samba/provider.yaml b/airflow/providers/samba/provider.yaml
new file mode 100644
index 0000000..8bf05a9
--- /dev/null
+++ b/airflow/providers/samba/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-samba
+description: |
+ `Samba <https://www.samba.org/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/segment/provider.yaml b/airflow/providers/segment/provider.yaml
new file mode 100644
index 0000000..a73ee8a
--- /dev/null
+++ b/airflow/providers/segment/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-segment
+description: |
+ `Segment <https://segment.com/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/sendgrid/provider.yaml b/airflow/providers/sendgrid/provider.yaml
new file mode 100644
index 0000000..575f7b8
--- /dev/null
+++ b/airflow/providers/sendgrid/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-sendgrid
+description: |
+ `Sendgrid <https://sendgrid.com/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/sftp/provider.yaml b/airflow/providers/sftp/provider.yaml
new file mode 100644
index 0000000..b2e3958
--- /dev/null
+++ b/airflow/providers/sftp/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-sftp
+description: |
+ `SSH File Transfer Protocol (SFTP) <https://tools.ietf.org/wg/secsh/draft-ietf-secsh-filexfer/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/singularity/provider.yaml b/airflow/providers/singularity/provider.yaml
new file mode 100644
index 0000000..89cd226
--- /dev/null
+++ b/airflow/providers/singularity/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-singularity
+description: |
+ `Singularity <https://sylabs.io/guides/latest/user-guide/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/slack/provider.yaml b/airflow/providers/slack/provider.yaml
new file mode 100644
index 0000000..4b29884
--- /dev/null
+++ b/airflow/providers/slack/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-slack
+description: |
+ `Slack <https://slack.com/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/snowflake/provider.yaml b/airflow/providers/snowflake/provider.yaml
new file mode 100644
index 0000000..ec1e666
--- /dev/null
+++ b/airflow/providers/snowflake/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-snowflake
+description: |
+ `Snowflake <https://www.snowflake.com/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/sqlite/provider.yaml b/airflow/providers/sqlite/provider.yaml
new file mode 100644
index 0000000..b727ac7
--- /dev/null
+++ b/airflow/providers/sqlite/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-sqlite
+description: |
+ `SQLite <https://www.sqlite.org/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/ssh/provider.yaml b/airflow/providers/ssh/provider.yaml
new file mode 100644
index 0000000..9148a20
--- /dev/null
+++ b/airflow/providers/ssh/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-ssh
+description: |
+ `Secure Shell (SSH) <https://tools.ietf.org/html/rfc4251>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/vertica/provider.yaml b/airflow/providers/vertica/provider.yaml
new file mode 100644
index 0000000..e908fd7
--- /dev/null
+++ b/airflow/providers/vertica/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-vertica
+description: |
+ `Verticaa <https://www.vertica.com/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/yandex/provider.yaml b/airflow/providers/yandex/provider.yaml
new file mode 100644
index 0000000..f913015
--- /dev/null
+++ b/airflow/providers/yandex/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-yandex
+description: |
+ Yandex including `Yandex.Cloud <https://cloud.yandex.com/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/airflow/providers/zendesk/provider.yaml b/airflow/providers/zendesk/provider.yaml
new file mode 100644
index 0000000..8b711ba
--- /dev/null
+++ b/airflow/providers/zendesk/provider.yaml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-zendesk
+description: |
+ `Zendesk <https://www.zendesk.com/>`__
+
+versions:
+ - 1.0.0b1
diff --git a/breeze-complete b/breeze-complete
index b5587a2..c2b3060 100644
--- a/breeze-complete
+++ b/breeze-complete
@@ -118,7 +118,7 @@ pyupgrade
restrict-start_date
rst-backticks
setup-order
-setup-installation
+setup-extra-packages
shellcheck
sort-in-the-wild
stylelint
diff --git a/dev/provider.yaml.schema.json b/dev/provider.yaml.schema.json
new file mode 100644
index 0000000..ba7989c
--- /dev/null
+++ b/dev/provider.yaml.schema.json
@@ -0,0 +1,31 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "type": "object",
+ "properties": {
+ "package-name": {
+ "description": "Package name available under which the package is available in the PyPi repository.",
+ "type": "string"
+ },
+ "python-module": {
+ "description": "Information about the package in RST format",
+ "type": "string"
+ },
+ "description": {
+ "description": "Information about the package in RST format",
+ "type": "string"
+ },
+ "versions": {
+ "description": "List of available versions in Pypi",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "package-name",
+ "description",
+ "versions"
+ ]
+}
diff --git a/docs/autoapi_templates/index.rst b/docs/autoapi_templates/index.rst
index 3686f42..df581b7 100644
--- a/docs/autoapi_templates/index.rst
+++ b/docs/autoapi_templates/index.rst
@@ -431,3 +431,74 @@ All task log handlers are derived from :class:`~airflow.utils.log.file_task_hand
airflow/providers/elasticsearch/log/index
airflow/providers/google/cloud/log/index
airflow/providers/microsoft/azure/log/index
+
+Providers
+---------
+
+Third party integrations are in a separate package :mod:`airflow.providers`.
+
+.. toctree::
+ :includehidden:
+ :glob:
+ :maxdepth: 1
+
+ airflow/providers/amazon/index
+ airflow/providers/apache/cassandra/index
+ airflow/providers/apache/druid/index
+ airflow/providers/apache/hdfs/index
+ airflow/providers/apache/hive/index
+ airflow/providers/apache/kylin/index
+ airflow/providers/apache/livy/index
+ airflow/providers/apache/pig/index
+ airflow/providers/apache/pinot/index
+ airflow/providers/apache/spark/index
+ airflow/providers/apache/sqoop/index
+ airflow/providers/celery/index
+ airflow/providers/cloudant/index
+ airflow/providers/cncf/kubernetes/index
+ airflow/providers/databricks/index
+ airflow/providers/datadog/index
+ airflow/providers/dingding/index
+ airflow/providers/discord/index
+ airflow/providers/docker/index
+ airflow/providers/elasticsearch/index
+ airflow/providers/exasol/index
+ airflow/providers/facebook/index
+ airflow/providers/ftp/index
+ airflow/providers/google/index
+ airflow/providers/grpc/index
+ airflow/providers/hashicorp/index
+ airflow/providers/http/index
+ airflow/providers/imap/index
+ airflow/providers/jdbc/index
+ airflow/providers/jenkins/index
+ airflow/providers/jira/index
+ airflow/providers/microsoft/azure/index
+ airflow/providers/microsoft/mssql/index
+ airflow/providers/microsoft/winrm/index
+ airflow/providers/mongo/index
+ airflow/providers/mysql/index
+ airflow/providers/odbc/index
+ airflow/providers/openfaas/index
+ airflow/providers/opsgenie/index
+ airflow/providers/oracle/index
+ airflow/providers/pagerduty/index
+ airflow/providers/papermill/index
+ airflow/providers/plexus/index
+ airflow/providers/postgres/index
+ airflow/providers/presto/index
+ airflow/providers/qubole/index
+ airflow/providers/redis/index
+ airflow/providers/salesforce/index
+ airflow/providers/samba/index
+ airflow/providers/segment/index
+ airflow/providers/sendgrid/index
+ airflow/providers/sftp/index
+ airflow/providers/singularity/index
+ airflow/providers/slack/index
+ airflow/providers/snowflake/index
+ airflow/providers/sqlite/index
+ airflow/providers/ssh/index
+ airflow/providers/vertica/index
+ airflow/providers/yandex/index
+ airflow/providers/zendesk/index
diff --git a/docs/build_docs.py b/docs/build_docs.py
index 6f4acc1..31690af 100755
--- a/docs/build_docs.py
+++ b/docs/build_docs.py
@@ -616,12 +616,14 @@ def check_spelling() -> None:
"sphinxcontrib.jinja",
"docroles",
"removemarktransform",
+ 'providers_packages_ref',
]
with NamedTemporaryFile() as tmp_file:
build_cmd = [
"sphinx-build",
- "-W",
+ "-W", # turn warnings into errors
+ "-T", # show full traceback on exception
"-b", # builder to use
"spelling",
"-d", # path for the cached environment and doctree files
@@ -659,12 +661,13 @@ def build_sphinx_docs() -> None:
with NamedTemporaryFile() as tmp_file:
build_cmd = [
"sphinx-build",
+ "-T", # show full traceback on exception
+ "--color", # do emit colored output
"-b", # builder to use
"html",
"-d", # path for the cached environment and doctree files
"_build/doctrees",
- "--color", # do emit colored output
- "-w", # turn warnings into errors
+ "-w", # write warnings (and errors) to given file
tmp_file.name,
".", # path to documentation source files
"_build/html", # path to output directory
diff --git a/docs/conf.py b/docs/conf.py
index d00d7ff..790c8cd 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -93,6 +93,7 @@ extensions = [
'removemarktransform',
'sphinx_copybutton',
'redirects',
+ 'providers_packages_ref',
# First, generate redoc
'sphinxcontrib.redoc',
# Second, update redoc script
@@ -114,15 +115,9 @@ exclude_patterns: List[str] = [
# We have custom page - operators-and-hooks-ref.rst
'_api/airflow/providers/index.rst',
# Packages with subpackages
- "_api/airflow/providers/amazon/index.rst",
- "_api/airflow/providers/facebook/index.rst",
"_api/airflow/providers/microsoft/index.rst",
- "_api/airflow/providers/google/index.rst",
"_api/airflow/providers/apache/index.rst",
- "_api/airflow/providers/yandex/index.rst",
"_api/airflow/providers/cncf/index.rst",
- # Packages without operators
- "_api/airflow/providers/sendgrid",
# Templates or partials
'autoapi_templates',
'howto/operator/google/_partials',
@@ -138,9 +133,7 @@ def _get_rst_filepath_from_path(filepath: str):
elif os.path.isfile(filepath) and filepath.endswith('/__init__.py'):
result = filepath.rpartition("/")[0]
else:
- result = filepath.rpartition(
- ".",
- )[0]
+ result = filepath.rpartition(".")[0]
result += "/index.rst"
result = f"_api/{os.path.relpath(result, ROOT_DIR)}"
@@ -159,36 +152,6 @@ for path in glob(f"{ROOT_DIR}/airflow/*"):
if os.path.isdir(path) and name not in browsable_packages:
exclude_patterns.append(f"_api/airflow/{name}")
-# Exclude package index
-_providers_packages_roots = {
- name.rpartition("/")[0]
- for entity in ["hooks", "operators", "secrets", "sensors"]
- for name in glob(f"{ROOT_DIR}/airflow/providers/**/{entity}", recursive=True)
-}
-
-_providers_package_indexes = {
- f"_api/{os.path.relpath(name, ROOT_DIR)}/index.rst" for name in _providers_packages_roots
-}
-
-exclude_patterns.extend(_providers_package_indexes)
-
-# Exclude auth_backend, utils, _internal_client, example_dags in providers packages
-_excluded_packages_in_providers = {
- name
- for entity in ['auth_backend', 'utils', '_internal_client', 'example_dags']
- for name in glob(f"{ROOT_DIR}/airflow/providers/**/{entity}/", recursive=True)
-}
-_excluded_files_in_providers = {
- _get_rst_filepath_from_path(path)
- for p in _excluded_packages_in_providers
- for path in glob(f"{p}/**/*", recursive=True)
-}
-_excluded_files_in_providers |= {
- _get_rst_filepath_from_path(name) for name in _excluded_packages_in_providers
-}
-
-exclude_patterns.extend(_excluded_files_in_providers)
-
# Add any paths that contain templates here, relative to this directory.
templates_path = ['templates']
@@ -421,8 +384,9 @@ autoapi_template_dir = 'autoapi_templates'
autoapi_ignore = [
'*/airflow/kubernetes/kubernetes_request_factory/*',
'*/_internal*',
+ '*/airflow/**/providers/**/utils/*',
'*/node_modules/*',
- '*/example_dags/*,',
+ '*/example_dags/*',
'*/migrations/*',
]
# Keep the AutoAPI generated files on the filesystem after the run.
diff --git a/docs/installation.rst b/docs/extra-packages-ref.rst
similarity index 68%
copy from docs/installation.rst
copy to docs/extra-packages-ref.rst
index 2c87b12..ffd02de 100644
--- a/docs/installation.rst
+++ b/docs/extra-packages-ref.rst
@@ -15,114 +15,10 @@
specific language governing permissions and limitations
under the License.
-.. contents:: :local:
+Extra Packages Reference
+''''''''''''''''''''''''
-Installation
-------------
-
-Getting Airflow
-'''''''''''''''
-
-Airflow is published as ``apache-airflow`` package in PyPI. Installing it however might be sometimes tricky
-because Airflow is a bit of both a library and application. Libraries usually keep their dependencies open and
-applications usually pin them, but we should do neither and both at the same time. We decided to keep
-our dependencies as open as possible (in ``setup.py``) so users can install different version of libraries
-if needed. This means that from time to time plain ``pip install apache-airflow`` will not work or will
-produce unusable Airflow installation.
-
-In order to have repeatable installation, however, starting from **Airflow 1.10.10** and updated in
-**Airflow 1.10.12** we also keep a set of "known-to-be-working" constraint files in the
-``constraints-master`` and ``constraints-1-10`` orphan branches.
-Those "known-to-be-working" constraints are per major/minor python version. You can use them as constraint
-files when installing Airflow from PyPI. Note that you have to specify correct Airflow version
-and python versions in the URL.
-
- **Prerequisites**
-
- On Debian based Linux OS:
-
- .. code-block:: bash
-
- sudo apt-get update
- sudo apt-get install build-essential
-
-
-1. Installing just airflow
-
-.. code-block:: bash
-
- AIRFLOW_VERSION=1.10.12
- PYTHON_VERSION="$(python --version | cut -d " " -f 2 | cut -d "." -f 1-2)"
- # For example: 3.6
- CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION}.txt"
- # For example: https://raw.githubusercontent.com/apache/airflow/constraints-1.10.12/constraints-3.6.txt
- pip install "apache-airflow==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}"
-
-2. Installing with extras (for example postgres, google)
-
-.. code-block:: bash
-
- AIRFLOW_VERSION=1.10.12
- PYTHON_VERSION="$(python --version | cut -d " " -f 2 | cut -d "." -f 1-2)"
- CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION}.txt"
- pip install "apache-airflow[postgres,google]==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}"
-
-Most of the extras are linked to a corresponding providers package. For example "amazon" extra
-has a corresponding ``apache-airflow-providers-amazon`` providers package to be installed. When you install
-Airflow with such extras, the necessary provider packages are installed automatically (latest versions from
-PyPI for those packages). However you can freely upgrade and install provider packages independently from
-the main Airflow installation.
-
-.. note:: Automated installation of Provider packages does not work in Airflow 2.0.0b1 - for this version
- you have to install provider packages manually. As of Airflow 2.0.0b2 the corresponding
- provider packages are installed together with the extras.
-
-Read more about it in the `Provider Packages <#provider-packages>`_ section.
-
-Requirements
-''''''''''''
-
-You need certain system level requirements in order to install Airflow. Those are requirements that are known
-to be needed for Linux system (Tested on Ubuntu Buster LTS) :
-
-.. code-block:: bash
-
- sudo apt-get install -y --no-install-recommends \
- freetds-bin \
- krb5-user \
- ldap-utils \
- libffi6 \
- libsasl2-2 \
- libsasl2-modules \
- libssl1.1 \
- locales \
- lsb-release \
- sasl2-bin \
- sqlite3 \
- unixodbc
-
-You also need database client packages (Postgres or MySQL) if you want to use those databases.
-
-If the ``airflow`` command is not getting recognized (can happen on Windows when using WSL), then
-ensure that ``~/.local/bin`` is in your ``PATH`` environment variable, and add it in if necessary:
-
-.. code-block:: bash
-
- PATH=$PATH:~/.local/bin
-
-Extra Packages
-''''''''''''''
-
-The ``apache-airflow`` PyPI basic package only installs what's needed to get started.
-Subpackages can be installed depending on what will be useful in your
-environment. For instance, if you don't need connectivity with Postgres,
-you won't have to go through the trouble of installing the ``postgres-devel``
-yum package, or whatever equivalent applies on the distribution you are using.
-
-Behind the scenes, Airflow does conditional imports of operators that require
-these extra dependencies.
-
-Here's the list of the subpackages and what they enable:
+Here's the list of the :ref:`subpackages <installation:extra_packages>` and what they enable:
**Fundamentals:**
@@ -302,121 +198,3 @@ Here's the list of the subpackages and what they enable:
+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
| winrm | ``pip install 'apache-airflow[microsoft.winrm]'`` | WinRM hooks and operators |
+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-
-Provider packages
------------------
-
-Provider packages context
-'''''''''''''''''''''''''
-
-Unlike Apache Airflow 1.10, the Airflow 2.0 is delivered in multiple, separate, but connected packages.
-The core of Airflow scheduling system is delivered as ``apache-airflow`` package and there are around
-60 providers packages which can be installed separately as so called "Airflow Provider packages".
-Those provider packages are separated per-provider (for example ``amazon``, ``google``, ``salesforce``
-etc.) Those packages are available as ``apache-airflow-providers`` packages - separately per each provider
-(for example there is an ``apache-airflow-providers-amazon`` or ``apache-airflow-providers-google`` package.
-
-You can install those provider packages separately in order to interface with a given provider. For those
-providers that have corresponding extras, the provider packages (latest version from PyPI) are installed
-automatically when Airflow is installed with the extra.
-
-Providers are released and versioned separately from the Airflow releases. We are following the
-`Semver <https://semver.org/>`_ versioning scheme for the packages. Some versions of the provider
-packages might depend on particular versions of Airflow, but the general approach we have is that unless
-there is a good reason, new version of providers should work with recent versions of Airflow 2.x. Details
-will vary per-provider and if there is a limitation for particular version of particular provider,
-constraining the Airflow version used, it will be included as limitation of dependencies in the provider
-package.
-
-Some of the providers have cross-provider dependencies as well. Those are not required dependencies, they
-might simply enable certain features (for example transfer operators often create dependency between
-different providers. Again, the general approach here is that the providers are backwards compatible,
-including cross-dependencies. Any kind of breaking changes and requirements on particular versions of other
-provider packages are automatically documented in the release notes of every provider.
-
-.. note::
- We also provide ``apache-airflow-backport-providers`` packages that can be installed for Airflow 1.10.
- Those are the same providers as for 2.0 but automatically back-ported to work for Airflow 1.10. Those
- backport providers are going to be updated and released for 3 months after Apache Airflow 2.0 release.
-
-Provider packages functionality
-'''''''''''''''''''''''''''''''
-
-Separate provider packages provide the possibilities that were not available in 1.10:
-
-1. You can upgrade to latest version of particular providers without the need of Apache Airflow core upgrade.
-
-2. You can downgrade to previous version of particular provider in case the new version introduces
- some problems, without impacting the main Apache Airflow core package.
-
-3. You can release and upgrade/downgrade provider packages incrementally, independent from each other. This
- means that you can incrementally validate each of the provider package update in your environment,
- following the usual tests you have in your environment.
-
-
-Q&A for Airflow and Providers
------------------------------
-
-Upgrading Airflow 2.0 and Providers
-'''''''''''''''''''''''''''''''''''
-
-Q. **When upgrading to a new Airflow version such as 2.0, but possibly 2.0.1 and beyond, is the best practice
- to also upgrade provider packages at the same time?**
-
-A. It depends on your use case. If you have automated or semi-automated verification of your installation,
- that you can run a new version of Airflow including all provider packages, then definitely go for it.
- If you rely more on manual testing, it is advised that you upgrade in stages. Depending on your choice
- you can either upgrade all used provider packages first, and then upgrade Airflow Core or the other way
- round. The first approach - when you first upgrade all providers is probably safer, as you can do it
- incrementally, step-by-step replacing provider by provider in your environment.
-
-Using Backport Providers in Airflow 1.10
-''''''''''''''''''''''''''''''''''''''''
-
-Q. **I have an Airflow version (1.10.12) running and it is stable. However, because of a Cloud provider change,
- I would like to upgrade the provider package. If I don't need to upgrade the Airflow version anymore,
- how do I know that this provider version is compatible with my Airflow version?**
-
-
-A. Backport Provider Packages (those are needed in 1.10.* Airflow series) are going to be released for
- 3 months after the release. We will stop releasing new updates to the backport providers afterwards.
- You will be able to continue using the provider packages that you already use and unless you need to
- get some new release of the provider that is only released for 2.0, there is no need to upgrade
- Airflow. This might happen if for example the provider is migrated to use newer version of client
- libraries or when new features/operators/hooks are added to it. Those changes will only be
- backported to 1.10.* compatible backport providers up to 3 months after releasing Airflow 2.0.
- Also we expect more providers, changes and fixes added to the existing providers to come after the
- 3 months pass. Eventually you will have to upgrade to Airflow 2.0 if you would like to make use of those.
- When it comes to compatibility of providers with different Airflow 2 versions, each
- provider package will keep its own dependencies, and while we expect those providers to be generally
- backwards-compatible, particular versions of particular providers might introduce dependencies on
- specific Airflow versions.
-
-Customizing Provider Packages
-'''''''''''''''''''''''''''''
-
-Q. **I have an older version of my provider package which we have lightly customized and is working
- fine with my MSSQL installation. I am upgrading my Airflow version. Do I need to upgrade my provider,
- or can I keep it as it is.**
-
-A. It depends on the scope of customization. There is no need to upgrade the provider packages to later
- versions unless you want to upgrade to Airflow version that introduces backwards-incompatible changes.
- Generally speaking, with Airflow 2 we are following the `Semver <https://semver.org/>`_ approach where
- we will introduce backwards-incompatible changes in Major releases, so all your modifications (as long
- as you have not used internal Airflow classes) should work for All Airflow 2.* versions.
-
-
-Initializing Airflow Database
------------------------------
-
-Airflow requires a database to be initialized before you can run tasks. If
-you're just experimenting and learning Airflow, you can stick with the
-default SQLite option. If you don't want to use SQLite, then take a look at
-:doc:`howto/initialize-database` to setup a different database.
-
-After configuration, you'll need to initialize the database before you can
-run tasks:
-
-.. code-block:: bash
-
- airflow db init
diff --git a/docs/exts/providers_packages_ref.py b/docs/exts/providers_packages_ref.py
new file mode 100644
index 0000000..3d8d8b9
--- /dev/null
+++ b/docs/exts/providers_packages_ref.py
@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+import os
+from glob import glob
+from typing import Any, Dict
+
+import jsonschema
+import yaml
+from sphinx.application import Sphinx
+
+ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
+PROVIDER_DATA_SCHEMA_PATH = os.path.join(ROOT_DIR, "dev", "provider.yaml.schema.json")
+
+
+def _load_schema() -> Dict[str, Any]:
+ with open(PROVIDER_DATA_SCHEMA_PATH) as schema_file:
+ content = json.load(schema_file)
+ return content
+
+
+def _filepath_to_module(filepath: str):
+ filepath = os.path.relpath(os.path.abspath(filepath), ROOT_DIR)
+ return filepath.replace("/", ".")
+
+
+def _load_package_data():
+ schema = _load_schema()
+ result = []
+ for provider_yaml_path in sorted(glob(f"{ROOT_DIR}/airflow/providers/**/provider.yaml")):
+ with open(provider_yaml_path) as yaml_file:
+ provider = yaml.safe_load(yaml_file)
+ provider['python-module'] = _filepath_to_module(os.path.dirname(provider_yaml_path))
+ try:
+ jsonschema.validate(provider, schema=schema)
+ except jsonschema.ValidationError:
+ raise Exception(f"Unable to parse: {provider_yaml_path}.")
+ result.append(provider)
+ return result
+
+
+def _on_config_inited(app, config):
+ del app
+ jinja_context = getattr(config, 'jinja_contexts', None) or {}
+
+ jinja_context['providers_ctx'] = {'providers': _load_package_data()}
+
+ config.jinja_contexts = jinja_context
+
+
+def setup(app: Sphinx):
+ """Setup plugin"""
+ app.setup_extension('sphinxcontrib.jinja')
+ app.connect("config-inited", _on_config_inited)
+
+ return {'parallel_read_safe': True, 'parallel_write_safe': True}
diff --git a/docs/exts/sphinx_script_update.py b/docs/exts/sphinx_script_update.py
index e5d6062..114287b 100644
--- a/docs/exts/sphinx_script_update.py
+++ b/docs/exts/sphinx_script_update.py
@@ -20,6 +20,7 @@ import os
import sys
import tempfile
from distutils.file_util import copy_file
+from functools import lru_cache
from typing import Dict
import requests
@@ -48,6 +49,7 @@ def _user_cache_dir(appname=None):
return path
+@lru_cache(maxsize=None)
def fetch_and_cache(script_url: str, output_filename: str):
"""Fetch URL to local cache and returns path."""
cache_key = _gethash(script_url)
@@ -89,10 +91,8 @@ def fetch_and_cache(script_url: str, output_filename: str):
return cache_filepath
-def build_finished(app, exception):
- """Sphinx "build_finished" event handler."""
- if exception:
- return
+def builder_inited(app):
+ """Sphinx "builder-inited" event handler."""
if not isinstance(app.builder, builders.StandaloneHTMLBuilder):
log.warning(
F"The plugin is support only 'html' builder, but you are using '{type(app.builder)}'. Skipping..."
@@ -101,6 +101,16 @@ def build_finished(app, exception):
script_url = app.config.redoc_script_url
output_filename = "script.js"
+ fetch_and_cache(script_url, output_filename)
+
+
+def build_finished(app, exception):
+ """Sphinx "build-finished" event handler."""
+ if exception or not isinstance(app.builder, builders.StandaloneHTMLBuilder):
+ return
+ script_url = app.config.redoc_script_url
+ output_filename = "script.js"
+
cache_filepath = fetch_and_cache(script_url, output_filename)
copy_file(cache_filepath, os.path.join(app.builder.outdir, '_static', "redoc.js"))
@@ -108,4 +118,5 @@ def build_finished(app, exception):
def setup(app):
"""Setup plugin"""
app.add_config_value("redoc_script_url", None, "env")
+ app.connect("builder-inited", builder_inited)
app.connect("build-finished", build_finished)
diff --git a/docs/index.rst b/docs/index.rst
index f1888d5..46678b3 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -78,6 +78,7 @@ Content
license
start
installation
+ provider-packages
tutorial
tutorial_taskflow_api
howto/index
@@ -115,3 +116,5 @@ Content
Experimental REST API <rest-api-ref>
Stable REST API <stable-rest-api-ref>
Configurations <configurations-ref>
+ Extra packages <extra-packages-ref>
+ Provider packages <provider-packages-ref>
diff --git a/docs/installation.rst b/docs/installation.rst
index 2c87b12..32b4688 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -15,11 +15,12 @@
specific language governing permissions and limitations
under the License.
-.. contents:: :local:
Installation
------------
+.. contents:: :local:
+
Getting Airflow
'''''''''''''''
@@ -110,6 +111,8 @@ ensure that ``~/.local/bin`` is in your ``PATH`` environment variable, and add i
PATH=$PATH:~/.local/bin
+.. _installation:extra_packages:
+
Extra Packages
''''''''''''''
@@ -122,292 +125,21 @@ yum package, or whatever equivalent applies on the distribution you are using.
Behind the scenes, Airflow does conditional imports of operators that require
these extra dependencies.
-Here's the list of the subpackages and what they enable:
-
-
-**Fundamentals:**
-
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| subpackage | install command | enables |
-+=====================+=====================================================+======================================================================+
-| all | ``pip install 'apache-airflow[all]'`` | All Airflow features known to man |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| all_dbs | ``pip install 'apache-airflow[all_dbs]'`` | All databases integrations |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| devel | ``pip install 'apache-airflow[devel]'`` | Minimum dev tools requirements |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| devel_hadoop | ``pip install 'apache-airflow[devel_hadoop]'`` | Airflow + dependencies on the Hadoop stack |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| doc | ``pip install 'apache-airflow[doc]'`` | Packages needed to build docs |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| password | ``pip install 'apache-airflow[password]'`` | Password authentication for users |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-
-
-**Apache Software:**
-
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| subpackage | install command | enables |
-+=====================+=====================================================+======================================================================+
-| atlas | ``pip install 'apache-airflow[apache.atlas]'`` | Apache Atlas to use Data Lineage feature |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| beam | ``pip install 'apache-airflow[apache.beam]'`` | Apache Beam operators & hooks |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| cassandra | ``pip install 'apache-airflow[apache.cassandra]'`` | Cassandra related operators & hooks |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| druid | ``pip install 'apache-airflow[apache.druid]'`` | Druid related operators & hooks |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| hdfs | ``pip install 'apache-airflow[apache.hdfs]'`` | HDFS hooks and operators |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| hive | ``pip install 'apache-airflow[apache.hive]'`` | All Hive related operators |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| kylin | ``pip install 'apache-airflow[apache.kylin]'`` | All Kylin related operators & hooks |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| spark | ``pip install 'apache-airflow[apache.spark]'`` | All Spark related operators & hooks |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| webhdfs | ``pip install 'apache-airflow[webhdfs]'`` | HDFS hooks and operators |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-
-
-**Services:**
-
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| subpackage | install command | enables |
-+=====================+=====================================================+======================================================================+
-| aws | ``pip install 'apache-airflow[amazon]'`` | Amazon Web Services |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| azure | ``pip install 'apache-airflow[microsoft.azure]'`` | Microsoft Azure |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| cloudant | ``pip install 'apache-airflow[cloudant]'`` | Cloudant hook |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| databricks | ``pip install 'apache-airflow[databricks]'`` | Databricks hooks and operators |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| datadog | ``pip install 'apache-airflow[datadog]'`` | Datadog hooks and sensors |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| facebook | ``pip install 'apache-airflow[facebook]'`` | Facebook Social |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| gcp | ``pip install 'apache-airflow[google]'`` | Google Cloud |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| github_enterprise | ``pip install 'apache-airflow[github_enterprise]'`` | GitHub Enterprise auth backend |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| google_auth | ``pip install 'apache-airflow[google_auth]'`` | Google auth backend |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| hashicorp | ``pip install 'apache-airflow[hashicorp]'`` | Hashicorp Services (Vault) |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| jira | ``pip install 'apache-airflow[jira]'`` | Jira hooks and operators |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| pagerduty | ``pip install 'apache-airflow[pagerduty]'`` | Pagerduty hook |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| plexus | ``pip install 'apache-airflow[plexus]'`` | Plexus service of CoreScientific.com AI platform |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| qds | ``pip install 'apache-airflow[qds]'`` | Enable QDS (Qubole Data Service) support |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| salesforce | ``pip install 'apache-airflow[salesforce]'`` | Salesforce hook |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| sendgrid | ``pip install 'apache-airflow[sendgrid]'`` | Send email using sendgrid |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| segment | ``pip install 'apache-airflow[segment]'`` | Segment hooks and sensors |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| sentry | ``pip install 'apache-airflow[sentry]'`` | Sentry service for application logging and monitoring |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| slack | ``pip install 'apache-airflow[slack]'`` | :class:`airflow.providers.slack.operators.slack.SlackAPIOperator` |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| snowflake | ``pip install 'apache-airflow[snowflake]'`` | Snowflake hooks and operators |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| vertica | ``pip install 'apache-airflow[vertica]'`` | Vertica hook support as an Airflow backend |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| yandexcloud | ``pip install 'apache-airflow[yandexcloud]'`` | Yandex.Cloud hooks and operators |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-
-
-**Software:**
-
-+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-| subpackage | install command | enables |
-+=====================+=====================================================+====================================================================================+
-| async | ``pip install 'apache-airflow[async]'`` | Async worker classes for Gunicorn |
-+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-| celery | ``pip install 'apache-airflow[celery]'`` | CeleryExecutor |
-+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-| dask | ``pip install 'apache-airflow[dask]'`` | DaskExecutor |
-+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-| docker | ``pip install 'apache-airflow[docker]'`` | Docker hooks and operators |
-+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-| elasticsearch | ``pip install 'apache-airflow[elasticsearch]'`` | Elasticsearch hooks and Log Handler |
-+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-| exasol | ``pip install 'apache-airflow[exasol]'`` | Exasol hooks and operators |
-+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-| kubernetes | ``pip install 'apache-airflow[cncf.kubernetes]'`` | Kubernetes Executor and operator |
-+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-| mongo | ``pip install 'apache-airflow[mongo]'`` | Mongo hooks and operators |
-+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-| mssql (deprecated) | ``pip install 'apache-airflow[microsoft.mssql]'`` | Microsoft SQL Server operators and hook, |
-| | | support as an Airflow backend. Uses pymssql. |
-| | | Will be replaced by subpackage ``odbc``. |
-+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-| mysql | ``pip install 'apache-airflow[mysql]'`` | MySQL operators and hook, support as an Airflow |
-| | | backend. The version of MySQL server has to be |
-| | | 5.6.4+. The exact version upper bound depends |
-| | | on version of ``mysqlclient`` package. For |
-| | | example, ``mysqlclient`` 1.3.12 can only be |
-| | | used with MySQL server 5.6.4 through 5.7. |
-+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-| odbc | ``pip install 'apache-airflow[odbc]'`` | ODBC data sources including MS SQL Server. Can use MsSqlOperator, |
-| | | or as metastore database backend. Uses pyodbc. |
-| | | See :ref:`howto/connection/odbc` for more info. |
-+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-| oracle | ``pip install 'apache-airflow[oracle]'`` | Oracle hooks and operators |
-+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-| pinot | ``pip install 'apache-airflow[pinot]'`` | Pinot DB hook |
-+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-| postgres | ``pip install 'apache-airflow[postgres]'`` | PostgreSQL operators and hook, support as an |
-| | | Airflow backend |
-+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-| presto | ``pip install 'apache-airflow[presto]'`` | All Presto related operators & hooks |
-+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-| rabbitmq | ``pip install 'apache-airflow[rabbitmq]'`` | RabbitMQ support as a Celery backend |
-+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-| redis | ``pip install 'apache-airflow[redis]'`` | Redis hooks and sensors |
-+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-| samba | ``pip install 'apache-airflow[samba]'`` | :class:`airflow.providers.apache.hive.transfers.hive_to_samba.HiveToSambaOperator` |
-+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-| singularity | ``pip install 'apache-airflow[singularity]'`` | Singularity container operator |
-+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-| statsd | ``pip install 'apache-airflow[statsd]'`` | Needed by StatsD metrics |
-+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-| tableau | ``pip install 'apache-airflow[tableau]'`` | Tableau visualization integration |
-+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-| virtualenv | ``pip install 'apache-airflow[virtualenv]'`` | Running python tasks in local virtualenv |
-+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-
-
-**Other:**
-
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| subpackage | install command | enables |
-+=====================+=====================================================+======================================================================+
-| cgroups | ``pip install 'apache-airflow[cgroups]'`` | Needed To use CgroupTaskRunner |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| grpc | ``pip install 'apache-airflow[grpc]'`` | Grpc hooks and operators |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| jdbc | ``pip install 'apache-airflow[jdbc]'`` | JDBC hooks and operators |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| kerberos | ``pip install 'apache-airflow[kerberos]'`` | Kerberos integration for Kerberized Hadoop |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| ldap | ``pip install 'apache-airflow[ldap]'`` | LDAP authentication for users |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| papermill | ``pip install 'apache-airflow[papermill]'`` | Papermill hooks and operators |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| ssh | ``pip install 'apache-airflow[ssh]'`` | SSH hooks and Operator |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| winrm | ``pip install 'apache-airflow[microsoft.winrm]'`` | WinRM hooks and operators |
-+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+For the list of the subpackages and what they enable, see: :doc:`extra-packages-ref`.
Provider packages
------------------
-
-Provider packages context
-'''''''''''''''''''''''''
+'''''''''''''''''
Unlike Apache Airflow 1.10, the Airflow 2.0 is delivered in multiple, separate, but connected packages.
The core of Airflow scheduling system is delivered as ``apache-airflow`` package and there are around
60 providers packages which can be installed separately as so called "Airflow Provider packages".
-Those provider packages are separated per-provider (for example ``amazon``, ``google``, ``salesforce``
-etc.) Those packages are available as ``apache-airflow-providers`` packages - separately per each provider
-(for example there is an ``apache-airflow-providers-amazon`` or ``apache-airflow-providers-google`` package.
-
-You can install those provider packages separately in order to interface with a given provider. For those
-providers that have corresponding extras, the provider packages (latest version from PyPI) are installed
-automatically when Airflow is installed with the extra.
-
-Providers are released and versioned separately from the Airflow releases. We are following the
-`Semver <https://semver.org/>`_ versioning scheme for the packages. Some versions of the provider
-packages might depend on particular versions of Airflow, but the general approach we have is that unless
-there is a good reason, new version of providers should work with recent versions of Airflow 2.x. Details
-will vary per-provider and if there is a limitation for particular version of particular provider,
-constraining the Airflow version used, it will be included as limitation of dependencies in the provider
-package.
-
-Some of the providers have cross-provider dependencies as well. Those are not required dependencies, they
-might simply enable certain features (for example transfer operators often create dependency between
-different providers. Again, the general approach here is that the providers are backwards compatible,
-including cross-dependencies. Any kind of breaking changes and requirements on particular versions of other
-provider packages are automatically documented in the release notes of every provider.
-
-.. note::
- We also provide ``apache-airflow-backport-providers`` packages that can be installed for Airflow 1.10.
- Those are the same providers as for 2.0 but automatically back-ported to work for Airflow 1.10. Those
- backport providers are going to be updated and released for 3 months after Apache Airflow 2.0 release.
-
-Provider packages functionality
-'''''''''''''''''''''''''''''''
-
-Separate provider packages provide the possibilities that were not available in 1.10:
-
-1. You can upgrade to latest version of particular providers without the need of Apache Airflow core upgrade.
-
-2. You can downgrade to previous version of particular provider in case the new version introduces
- some problems, without impacting the main Apache Airflow core package.
-
-3. You can release and upgrade/downgrade provider packages incrementally, independent from each other. This
- means that you can incrementally validate each of the provider package update in your environment,
- following the usual tests you have in your environment.
-
-
-Q&A for Airflow and Providers
------------------------------
-
-Upgrading Airflow 2.0 and Providers
-'''''''''''''''''''''''''''''''''''
-
-Q. **When upgrading to a new Airflow version such as 2.0, but possibly 2.0.1 and beyond, is the best practice
- to also upgrade provider packages at the same time?**
-
-A. It depends on your use case. If you have automated or semi-automated verification of your installation,
- that you can run a new version of Airflow including all provider packages, then definitely go for it.
- If you rely more on manual testing, it is advised that you upgrade in stages. Depending on your choice
- you can either upgrade all used provider packages first, and then upgrade Airflow Core or the other way
- round. The first approach - when you first upgrade all providers is probably safer, as you can do it
- incrementally, step-by-step replacing provider by provider in your environment.
-
-Using Backport Providers in Airflow 1.10
-''''''''''''''''''''''''''''''''''''''''
-
-Q. **I have an Airflow version (1.10.12) running and it is stable. However, because of a Cloud provider change,
- I would like to upgrade the provider package. If I don't need to upgrade the Airflow version anymore,
- how do I know that this provider version is compatible with my Airflow version?**
-
-
-A. Backport Provider Packages (those are needed in 1.10.* Airflow series) are going to be released for
- 3 months after the release. We will stop releasing new updates to the backport providers afterwards.
- You will be able to continue using the provider packages that you already use and unless you need to
- get some new release of the provider that is only released for 2.0, there is no need to upgrade
- Airflow. This might happen if for example the provider is migrated to use newer version of client
- libraries or when new features/operators/hooks are added to it. Those changes will only be
- backported to 1.10.* compatible backport providers up to 3 months after releasing Airflow 2.0.
- Also we expect more providers, changes and fixes added to the existing providers to come after the
- 3 months pass. Eventually you will have to upgrade to Airflow 2.0 if you would like to make use of those.
- When it comes to compatibility of providers with different Airflow 2 versions, each
- provider package will keep its own dependencies, and while we expect those providers to be generally
- backwards-compatible, particular versions of particular providers might introduce dependencies on
- specific Airflow versions.
-
-Customizing Provider Packages
-'''''''''''''''''''''''''''''
-
-Q. **I have an older version of my provider package which we have lightly customized and is working
- fine with my MSSQL installation. I am upgrading my Airflow version. Do I need to upgrade my provider,
- or can I keep it as it is.**
-
-A. It depends on the scope of customization. There is no need to upgrade the provider packages to later
- versions unless you want to upgrade to Airflow version that introduces backwards-incompatible changes.
- Generally speaking, with Airflow 2 we are following the `Semver <https://semver.org/>`_ approach where
- we will introduce backwards-incompatible changes in Major releases, so all your modifications (as long
- as you have not used internal Airflow classes) should work for All Airflow 2.* versions.
+The default Airflow installation doesn't have many integrations and you have to install them yourself.
+For more information, see: :doc:`provider-packages`
+For the list of the provider packages and what they enable, see: :doc:`provider-packages-ref`.
Initializing Airflow Database
------------------------------
+'''''''''''''''''''''''''''''
Airflow requires a database to be initialized before you can run tasks. If
you're just experimenting and learning Airflow, you can stick with the
@@ -420,3 +152,8 @@ run tasks:
.. code-block:: bash
airflow db init
+
+Docker image
+''''''''''''
+
+Airflow is also distributed as a Docker image (OCI Image). For more information, see: :doc:`production-deployment`
diff --git a/docs/operators-and-hooks-ref.rst b/docs/operators-and-hooks-ref.rst
index 9df3868..4d9e958 100644
--- a/docs/operators-and-hooks-ref.rst
+++ b/docs/operators-and-hooks-ref.rst
@@ -1359,7 +1359,7 @@ These integrations allow you to perform various operations within various servic
- :mod:`airflow.providers.salesforce.operators.tableau_refresh_workbook`
- :mod:`airflow.providers.salesforce.sensors.tableau_job_status`
- * - `Segment <https://oapi.dingtalk.com>`__
+ * - `Segment <https://segment.com/>`__
-
- :mod:`airflow.providers.segment.hooks.segment`
- :mod:`airflow.providers.segment.operators.segment_track_event`
diff --git a/docs/provider-packages-ref.rst b/docs/provider-packages-ref.rst
new file mode 100644
index 0000000..c57a42e
--- /dev/null
+++ b/docs/provider-packages-ref.rst
@@ -0,0 +1,60 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+Providers packages reference
+''''''''''''''''''''''''''''
+
+Here's the list of the :doc:`provider packages <provider-packages>` and what they enable:
+
+
+.. contents:: :local:
+
+.. jinja:: providers_ctx
+
+ {% for package in providers %}
+
+ ``{{ package["package-name"] }}``
+ {{ "=" * (package["package-name"]|length + 4) }}
+
+ {% if package["description"] %}
+ {{ package["description"] }}
+ {% endif %}
+
+ {% if package["versions"] %}
+ To install, run:
+
+ .. code-block:: bash
+
+ pip install '{{ package["package-name"] }}'
+
+ :Available versions: {% for version in package["versions"] %}``{{ version }}``{% if not loop.last %}, {% else %}.{% endif %}{%- endfor %}
+
+ :Reference: `PyPi repository <https://pypi.org/project/{{ package["package-name"] }}/>`__
+ {% if package["python-module"] %}
+ :Python API Reference: :mod:`{{ package["python-module"] }}`
+ {% endif %}
+ {% else %}
+
+ .. warning::
+
+ This package has not yet been released.
+
+ {% if package["python-module"] %}
+ :Python API Reference: :mod:`{{ package["python-module"] }}`
+ {% endif %}
+ {% endif %}
+ {% endfor %}
diff --git a/docs/provider-packages.rst b/docs/provider-packages.rst
new file mode 100644
index 0000000..7b405ed
--- /dev/null
+++ b/docs/provider-packages.rst
@@ -0,0 +1,120 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+Provider packages
+-----------------
+
+.. contents:: :local:
+
+Provider packages context
+'''''''''''''''''''''''''
+
+Unlike Apache Airflow 1.10, the Airflow 2.0 is delivered in multiple, separate, but connected packages.
+The core of Airflow scheduling system is delivered as ``apache-airflow`` package and there are around
+60 providers packages which can be installed separately as so called "Airflow Provider packages".
+Those provider packages are separated per-provider (for example ``amazon``, ``google``, ``salesforce``
+etc.) Those packages are available as ``apache-airflow-providers`` packages - separately per each provider
+(for example there is an ``apache-airflow-providers-amazon`` or ``apache-airflow-providers-google`` package.
+
+You can install those provider packages separately in order to interface with a given provider. For those
+providers that have corresponding extras, the provider packages (latest version from PyPI) are installed
+automatically when Airflow is installed with the extra.
+
+Providers are released and versioned separately from the Airflow releases. We are following the
+`Semver <https://semver.org/>`_ versioning scheme for the packages. Some versions of the provider
+packages might depend on particular versions of Airflow, but the general approach we have is that unless
+there is a good reason, new version of providers should work with recent versions of Airflow 2.x. Details
+will vary per-provider and if there is a limitation for particular version of particular provider,
+constraining the Airflow version used, it will be included as limitation of dependencies in the provider
+package.
+
+Some of the providers have cross-provider dependencies as well. Those are not required dependencies, they
+might simply enable certain features (for example transfer operators often create dependency between
+different providers. Again, the general approach here is that the providers are backwards compatible,
+including cross-dependencies. Any kind of breaking changes and requirements on particular versions of other
+provider packages are automatically documented in the release notes of every provider.
+
+.. note::
+ We also provide ``apache-airflow-backport-providers`` packages that can be installed for Airflow 1.10.
+ Those are the same providers as for 2.0 but automatically back-ported to work for Airflow 1.10. Those
+ backport providers are going to be updated and released for 3 months after Apache Airflow 2.0 release.
+
+Provider packages functionality
+'''''''''''''''''''''''''''''''
+
+Separate provider packages provide the possibilities that were not available in 1.10:
+
+1. You can upgrade to latest version of particular providers without the need of Apache Airflow core upgrade.
+
+2. You can downgrade to previous version of particular provider in case the new version introduces
+ some problems, without impacting the main Apache Airflow core package.
+
+3. You can release and upgrade/downgrade provider packages incrementally, independent from each other. This
+ means that you can incrementally validate each of the provider package update in your environment,
+ following the usual tests you have in your environment.
+
+
+Q&A for Airflow and Providers
+'''''''''''''''''''''''''''''
+
+Upgrading Airflow 2.0 and Providers
+"""""""""""""""""""""""""""""""""""
+
+Q. **When upgrading to a new Airflow version such as 2.0, but possibly 2.0.1 and beyond, is the best practice
+ to also upgrade provider packages at the same time?**
+
+A. It depends on your use case. If you have automated or semi-automated verification of your installation,
+ that you can run a new version of Airflow including all provider packages, then definitely go for it.
+ If you rely more on manual testing, it is advised that you upgrade in stages. Depending on your choice
+ you can either upgrade all used provider packages first, and then upgrade Airflow Core or the other way
+ round. The first approach - when you first upgrade all providers is probably safer, as you can do it
+ incrementally, step-by-step replacing provider by provider in your environment.
+
+Using Backport Providers in Airflow 1.10
+""""""""""""""""""""""""""""""""""""""""
+
+Q. **I have an Airflow version (1.10.12) running and it is stable. However, because of a Cloud provider change,
+ I would like to upgrade the provider package. If I don't need to upgrade the Airflow version anymore,
+ how do I know that this provider version is compatible with my Airflow version?**
+
+
+A. Backport Provider Packages (those are needed in 1.10.* Airflow series) are going to be released for
+ 3 months after the release. We will stop releasing new updates to the backport providers afterwards.
+ You will be able to continue using the provider packages that you already use and unless you need to
+ get some new release of the provider that is only released for 2.0, there is no need to upgrade
+ Airflow. This might happen if for example the provider is migrated to use newer version of client
+ libraries or when new features/operators/hooks are added to it. Those changes will only be
+ backported to 1.10.* compatible backport providers up to 3 months after releasing Airflow 2.0.
+ Also we expect more providers, changes and fixes added to the existing providers to come after the
+ 3 months pass. Eventually you will have to upgrade to Airflow 2.0 if you would like to make use of those.
+ When it comes to compatibility of providers with different Airflow 2 versions, each
+ provider package will keep its own dependencies, and while we expect those providers to be generally
+ backwards-compatible, particular versions of particular providers might introduce dependencies on
+ specific Airflow versions.
+
+Customizing Provider Packages
+"""""""""""""""""""""""""""""
+
+Q. **I have an older version of my provider package which we have lightly customized and is working
+ fine with my MSSQL installation. I am upgrading my Airflow version. Do I need to upgrade my provider,
+ or can I keep it as it is.**
+
+A. It depends on the scope of customization. There is no need to upgrade the provider packages to later
+ versions unless you want to upgrade to Airflow version that introduces backwards-incompatible changes.
+ Generally speaking, with Airflow 2 we are following the `Semver <https://semver.org/>`_ approach where
+ we will introduce backwards-incompatible changes in Major releases, so all your modifications (as long
+ as you have not used internal Airflow classes) should work for All Airflow 2.* versions.
diff --git a/scripts/ci/pre_commit/pre_commit_check_setup_installation.py b/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py
similarity index 99%
rename from scripts/ci/pre_commit/pre_commit_check_setup_installation.py
rename to scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py
index 15479ea..b26e833 100755
--- a/scripts/ci/pre_commit/pre_commit_check_setup_installation.py
+++ b/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py
@@ -32,7 +32,7 @@ from rich.table import Table
AIRFLOW_SOURCES_DIR = os.path.join(dirname(__file__), os.pardir, os.pardir, os.pardir)
SETUP_PY_FILE = 'setup.py'
-DOCS_FILE = 'installation.rst'
+DOCS_FILE = 'extra-packages-ref.rst'
PY_IDENTIFIER = r'[a-zA-Z_][a-zA-Z0-9_\.]*'
diff --git a/setup.py b/setup.py
index 1f4eb22..fe2522c 100644
--- a/setup.py
+++ b/setup.py
@@ -383,6 +383,10 @@ slack = [
'slackclient>=2.0.0,<3.0.0',
]
snowflake = [
+ # snowflake is not compatible with latest version.
+ # This library monkey patches the requests library, so SSL is broken globally.
+ # See: https://github.com/snowflakedb/snowflake-connector-python/issues/324
+ 'requests<2.24.0',
'snowflake-connector-python>=1.5.2',
'snowflake-sqlalchemy>=1.1.0',
]