You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2023/11/15 02:37:58 UTC
(airflow) branch main updated: Remove all code related to Qubole provider (#35636)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 83257a7b35 Remove all code related to Qubole provider (#35636)
83257a7b35 is described below
commit 83257a7b350dcdd98bdcb7d35d37deef0379c97b
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Wed Nov 15 03:37:50 2023 +0100
Remove all code related to Qubole provider (#35636)
Following the discussion at the Airflow devlist:
https://lists.apache.org/thread/7wwz9lwtk6dp15qv179jso63f2xx54ot
We are removing Qubole provider after last release with "removal"
notice has been published.
The last release of Qubole Provider (3.4.3) has been released on
Nov 12, 2023 and is available on PyPI:
https://pypi.org/project/apache-airflow-providers-qubole/
No more releases of Qubole Provider will be made
---
.../airflow_providers_bug_report.yml | 1 -
.github/boring-cyborg.yml | 6 -
INTHEWILD.md | 1 -
airflow/contrib/hooks/__init__.py | 6 -
airflow/contrib/operators/__init__.py | 9 -
airflow/contrib/sensors/__init__.py | 5 -
.../providers/qubole/.latest-doc-only-change.txt | 1 -
airflow/providers/qubole/CHANGELOG.rst | 341 ------------------
airflow/providers/qubole/__init__.py | 43 ---
airflow/providers/qubole/hooks/__init__.py | 17 -
airflow/providers/qubole/hooks/qubole.py | 304 ----------------
airflow/providers/qubole/hooks/qubole_check.py | 122 -------
airflow/providers/qubole/operators/__init__.py | 17 -
airflow/providers/qubole/operators/qubole.py | 303 ----------------
airflow/providers/qubole/operators/qubole_check.py | 210 -----------
airflow/providers/qubole/provider.yaml | 85 -----
airflow/providers/qubole/sensors/__init__.py | 17 -
airflow/providers/qubole/sensors/qubole.py | 117 -------
airflow/utils/db.py | 8 -
dev/README_RELEASE_PROVIDER_PACKAGES.md | 20 +-
docs/apache-airflow-providers-qubole/changelog.rst | 18 -
docs/apache-airflow-providers-qubole/commits.rst | 390 ---------------------
docs/apache-airflow-providers-qubole/index.rst | 112 ------
.../installing-providers-from-sources.rst | 18 -
.../operators/index.rst | 32 --
.../operators/qubole.rst | 184 ----------
.../operators/qubole_check.rst | 45 ---
docs/apache-airflow-providers-qubole/security.rst | 19 -
docs/apache-airflow/extra-packages-ref.rst | 2 -
docs/integration-logos/qubole/Qubole.png | Bin 1892 -> 0 bytes
setup.py | 1 -
tests/providers/qubole/__init__.py | 17 -
tests/providers/qubole/hooks/__init__.py | 17 -
tests/providers/qubole/hooks/test_qubole.py | 80 -----
tests/providers/qubole/hooks/test_qubole_check.py | 42 ---
tests/providers/qubole/operators/__init__.py | 17 -
tests/providers/qubole/operators/test_qubole.py | 179 ----------
.../qubole/operators/test_qubole_check.py | 192 ----------
tests/providers/qubole/sensors/__init__.py | 17 -
tests/providers/qubole/sensors/test_qubole.py | 80 -----
tests/system/providers/qubole/__init__.py | 16 -
tests/system/providers/qubole/example_qubole.py | 235 -------------
.../providers/qubole/example_qubole_sensors.py | 94 -----
43 files changed, 10 insertions(+), 3430 deletions(-)
diff --git a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml
index f117380e72..053d913d6f 100644
--- a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml
+++ b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml
@@ -93,7 +93,6 @@ body:
- plexus
- postgres
- presto
- - qubole
- redis
- salesforce
- samba
diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml
index a1bd2351db..17a46a57cf 100644
--- a/.github/boring-cyborg.yml
+++ b/.github/boring-cyborg.yml
@@ -390,12 +390,6 @@ labelPRBasedOnFilePath:
- tests/providers/presto/**/*
- tests/system/providers/presto/**/*
- provider:qubole:
- - airflow/providers/qubole/**/*
- - docs/apache-airflow-providers-qubole/**/*
- - tests/providers/qubole/**/*
- - tests/system/providers/qubole/**/*
-
provider:redis:
- airflow/providers/redis/**/*
- docs/apache-airflow-providers-redis/**/*
diff --git a/INTHEWILD.md b/INTHEWILD.md
index e3450e9f1a..8e1740142a 100644
--- a/INTHEWILD.md
+++ b/INTHEWILD.md
@@ -379,7 +379,6 @@ Currently, **officially** using Airflow:
1. [Qoala](https://www.qoala.id) [[@gnomeria](https://github.com/gnomeria), [@qoala-engineering](https://github.com/qoala-engineering)]
1. [Qplum](https://qplum.co) [[@manti](https://github.com/manti)]
1. [Quantopian](https://www.quantopian.com/) [[@eronarn](https://github.com/eronarn)]
-1. [Qubole](https://qubole.com) [[@msumit](https://github.com/msumit)]
1. [Quick Algorithm Analytics](https://quickalgorithm.com) [[@QuickAlgorithm](https://github.com/QuickAlgorithm), [@jacojaki](https://github.com/jacojaki), [@Giuzzilla](https://github.com/Giuzzilla), [@breizh24](https://github.com/breizh24) & [@AndreaSoprani](https://github.com/andreasoprani)]
1. [QuintoAndar](https://quintoandar.com.br) [[@quintoandar](https://github.com/quintoandar)]
1. [Quizlet](https://quizlet.com) [[@quizlet](https://github.com/quizlet)]
diff --git a/airflow/contrib/hooks/__init__.py b/airflow/contrib/hooks/__init__.py
index 5385599aca..b2c7612989 100644
--- a/airflow/contrib/hooks/__init__.py
+++ b/airflow/contrib/hooks/__init__.py
@@ -267,12 +267,6 @@ __deprecated_classes = {
"PinotAdminHook": "airflow.providers.apache.pinot.hooks.pinot.PinotAdminHook",
"PinotDbApiHook": "airflow.providers.apache.pinot.hooks.pinot.PinotDbApiHook",
},
- "qubole_check_hook": {
- "QuboleCheckHook": "airflow.providers.qubole.hooks.qubole_check.QuboleCheckHook",
- },
- "qubole_hook": {
- "QuboleHook": "airflow.providers.qubole.hooks.qubole.QuboleHook",
- },
"redis_hook": {
"RedisHook": "airflow.providers.redis.hooks.redis.RedisHook",
},
diff --git a/airflow/contrib/operators/__init__.py b/airflow/contrib/operators/__init__.py
index fe090c8b0d..faf1899606 100644
--- a/airflow/contrib/operators/__init__.py
+++ b/airflow/contrib/operators/__init__.py
@@ -1049,15 +1049,6 @@ __deprecated_classes = {
"airflow.providers.google.cloud.operators.pubsub.PubSubDeleteTopicOperator"
),
},
- "qubole_check_operator": {
- "QuboleCheckOperator": "airflow.providers.qubole.operators.qubole_check.QuboleCheckOperator",
- "QuboleValueCheckOperator": (
- "airflow.providers.qubole.operators.qubole_check.QuboleValueCheckOperator"
- ),
- },
- "qubole_operator": {
- "QuboleOperator": "airflow.providers.qubole.operators.qubole.QuboleOperator",
- },
"redis_publish_operator": {
"RedisPublishOperator": "airflow.providers.redis.operators.redis_publish.RedisPublishOperator",
},
diff --git a/airflow/contrib/sensors/__init__.py b/airflow/contrib/sensors/__init__.py
index 9da2c36845..9d9d1c1557 100644
--- a/airflow/contrib/sensors/__init__.py
+++ b/airflow/contrib/sensors/__init__.py
@@ -130,11 +130,6 @@ __deprecated_classes = {
"python_sensor": {
"PythonSensor": "airflow.sensors.python.PythonSensor",
},
- "qubole_sensor": {
- "QuboleFileSensor": "airflow.providers.qubole.sensors.qubole.QuboleFileSensor",
- "QubolePartitionSensor": "airflow.providers.qubole.sensors.qubole.QubolePartitionSensor",
- "QuboleSensor": "airflow.providers.qubole.sensors.qubole.QuboleSensor",
- },
"redis_key_sensor": {
"RedisKeySensor": "airflow.providers.redis.sensors.redis_key.RedisKeySensor",
},
diff --git a/airflow/providers/qubole/.latest-doc-only-change.txt b/airflow/providers/qubole/.latest-doc-only-change.txt
deleted file mode 100644
index 5dcc5759cd..0000000000
--- a/airflow/providers/qubole/.latest-doc-only-change.txt
+++ /dev/null
@@ -1 +0,0 @@
-e45bee884068399e7265421511e17fed106ce5b4
diff --git a/airflow/providers/qubole/CHANGELOG.rst b/airflow/providers/qubole/CHANGELOG.rst
deleted file mode 100644
index 8369d59a76..0000000000
--- a/airflow/providers/qubole/CHANGELOG.rst
+++ /dev/null
@@ -1,341 +0,0 @@
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-
-.. NOTE TO CONTRIBUTORS:
- Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes
- and you want to add an explanation to the users on how they are supposed to deal with them.
- The changelog is updated and maintained semi-automatically by release manager.
-
-``apache-airflow-providers-qubole``
-
-
-Changelog
----------
-
-3.4.3
-.....
-
-.. warning::
-
- This is the last release of the provider.
-
- This provider is not maintained anymore by the community. It has been removed and is not going to be
- updated anymore. The removal was done according to the process described in
- `Removing community providers <https://github.com/apache/airflow/blob/main/PROVIDERS.rst#removing-community-providers>`_
-
- Feel free to contact Airflow Development Mailing List if you have any questions.
-
-Misc
-~~~~
-
-* ``Suspend qubole provider (#33889)``
-
-.. Review and move the new changes to one of the sections above:
- * ``Add information about Qubole removal and make it possible to release it (#35492)``
- * ``Switch from Black to Ruff formatter (#35287)``
- * ``Pre-upgrade 'ruff==0.0.292' changes in providers (#35053)``
- * ``Upgrade pre-commits (#35033)``
- * ``Prepare docs 1st wave of Providers in October 2023 (#34916)``
- * ``Bump min airflow version of providers (#34728)``
-
-3.4.2
-.....
-
-.. note::
- This release of provider is only available for Airflow 2.5+ as explained in the
- `Apache Airflow providers support policy <https://github.com/apache/airflow/blob/main/PROVIDERS.rst#minimum-supported-version-of-airflow-for-community-managed-providers>`_.
-
-Misc
-~~~~
-
-* ``Add a new parameter to SQL operators to specify conn id field (#30784)``
-
-.. Below changes are excluded from the changelog. Move them to
- appropriate section above if needed. Do not delete the lines(!):
- * ``Upgrade ruff to latest 0.0.282 version in pre-commits (#33152)``
- * ``Prepare docs for July 2023 wave of Providers (RC2) (#32381)``
- * ``D205 Support - Providers: Pagerduty to SMTP (inclusive) (#32358)``
- * ``Remove spurious headers for provider changelogs (#32373)``
- * ``Prepare docs for July 2023 wave of Providers (#32298)``
- * ``Improve provider documentation and README structure (#32125)``
-
-3.4.1
-.....
-
-.. note::
- This release dropped support for Python 3.7
-
-Misc
-~~~~
-
-* ``Add note about dropping Python 3.7 for providers (#32015)``
-
-.. Below changes are excluded from the changelog. Move them to
- appropriate section above if needed. Do not delete the lines(!):
- * ``Add D400 pydocstyle check - Providers (#31427)``
-
-3.4.0
-.....
-
-.. note::
- This release of provider is only available for Airflow 2.4+ as explained in the
- `Apache Airflow providers support policy <https://github.com/apache/airflow/blob/main/PROVIDERS.rst#minimum-supported-version-of-airflow-for-community-managed-providers>`_.
-
-Misc
-~~~~
-
-* ``Bump minimum Airflow version in providers (#30917)``
-
-.. Below changes are excluded from the changelog. Move them to
- appropriate section above if needed. Do not delete the lines(!):
- * ``Move TaskInstanceKey to a separate file (#31033)``
- * ``Decouple "job runner" from BaseJob ORM model (#30255)``
- * ``Add full automation for min Airflow version for providers (#30994)``
- * ``Add mechanism to suspend providers (#30422)``
- * ``Use '__version__' in providers not 'version' (#31393)``
- * ``Fixing circular import error in providers caused by airflow version check (#31379)``
- * ``Prepare docs for May 2023 wave of Providers (#31252)``
-
-3.3.1
-.....
-
-Bug Fixes
-~~~~~~~~~
-
-* ``Bump common.sql provider to 1.3.1 (#27888)``
-
-.. Below changes are excluded from the changelog. Move them to
- appropriate section above if needed. Do not delete the lines(!):
- * ``Prepare for follow-up release for November providers (#27774)``
-
-3.3.0
-.....
-
-.. note::
- This release of provider is only available for Airflow 2.3+ as explained in the
- `Apache Airflow providers support policy <https://github.com/apache/airflow/blob/main/PROVIDERS.rst#minimum-supported-version-of-airflow-for-community-managed-providers>`_.
-
-Misc
-~~~~
-
-* ``Move min airflow version to 2.3.0 for all providers (#27196)``
-
-.. Below changes are excluded from the changelog. Move them to
- appropriate section above if needed. Do not delete the lines(!):
- * ``Update old style typing (#26872)``
- * ``Enable string normalization in python formatting - providers (#27205)``
-
-3.2.1
-.....
-
-Misc
-~~~~
-
-* ``Add common-sql lower bound for common-sql (#25789)``
-
-.. Review and move the new changes to one of the sections above:
- * ``Apply PEP-563 (Postponed Evaluation of Annotations) to non-core airflow (#26289)``
- * ``D400 first line should end with period batch02 (#25268)``
-
-3.2.0
-.....
-
-Features
-~~~~~~~~
-
-* ``Make extra link work in UI (#25500)``
-* ``Move all "old" SQL operators to common.sql providers (#25350)``
-* ``Improve taskflow type hints with ParamSpec (#25173)``
-
-Bug Fixes
-~~~~~~~~~
-
-* ``Correctly render 'results_parser_callable' param in Qubole docs (#25514)``
-
-
-3.1.0
-.....
-
-Features
-~~~~~~~~
-
-* ``Move all SQL classes to common-sql provider (#24836)``
-
-
-.. Below changes are excluded from the changelog. Move them to
- appropriate section above if needed. Do not delete the lines(!):
- * ``Remove "bad characters" from our codebase (#24841)``
- * ``Move provider dependencies to inside provider folders (#24672)``
- * ``Remove 'hook-class-names' from provider.yaml (#24702)``
-
-3.0.0
-.....
-
-Breaking changes
-~~~~~~~~~~~~~~~~
-
-.. note::
- This release of provider is only available for Airflow 2.2+ as explained in the
- `Apache Airflow providers support policy <https://github.com/apache/airflow/blob/main/PROVIDERS.rst#minimum-supported-version-of-airflow-for-community-managed-providers>`_.
-
-Bug Fixes
-~~~~~~~~~
-
-* ``Add typing for airflow/configuration.py (#23716)``
-* ``Fix backwards-compatibility introduced by fixing mypy problems (#24230)``
-
-.. Below changes are excluded from the changelog. Move them to
- appropriate section above if needed. Do not delete the lines(!):
- * ``Migrate Qubole example DAGs to new design #22460 (#24149)``
- * ``Prepare provider documentation 2022.05.11 (#23631)``
- * ``Use new Breese for building, pulling and verifying the images. (#23104)``
- * ``Replace usage of 'DummyOperator' with 'EmptyOperator' (#22974)``
- * ``Add explanatory note for contributors about updating Changelog (#24229)``
- * ``Prepare docs for May 2022 provider's release (#24231)``
- * ``Update package description to remove double min-airflow specification (#24292)``
-
-2.1.3
-.....
-
-Bug Fixes
-~~~~~~~~~
-
-* ``Fix mistakenly added install_requires for all providers (#22382)``
-
-2.1.2
-.....
-
-Misc
-~~~~~
-
-* ``Add Trove classifiers in PyPI (Framework :: Apache Airflow :: Provider)``
-
-.. Below changes are excluded from the changelog. Move them to
- appropriate section above if needed. Do not delete the lines(!):
- * `` Add map_index to XCom model and interface (#22112)``
- * ``Protect against accidental misuse of XCom.get_value() (#22244)``
-
-2.1.1
-.....
-
-Misc
-~~~~
-
-* ``Support for Python 3.10``
-
-.. Below changes are excluded from the changelog. Move them to
- appropriate section above if needed. Do not delete the lines(!):
- * ``Change BaseOperatorLink interface to take a ti_key, not a datetime (#21798)``
-
-2.1.0
-.....
-
-Features
-~~~~~~~~
-
-* ``Add optional features in providers. (#21074)``
-
-Misc
-~~~~
-
-* ``Add Qubole how to documentation (#20058)``
-
-.. Below changes are excluded from the changelog. Move them to
- appropriate section above if needed. Do not delete the lines(!):
- * ``Fix K8S changelog to be PyPI-compatible (#20614)``
- * ``Fix template_fields type to have MyPy friendly Sequence type (#20571)``
- * ``Fix MyPy Errors for Qubole provider. (#20319)``
- * ``Refactor operator links to not create ad hoc TaskInstances (#21285)``
- * ``Remove ':type' lines now sphinx-autoapi supports typehints (#20951)``
- * ``Fxing MyPy issues inside airflow/providers/qubole (#20625)``
- * ``Update documentation for provider December 2021 release (#20523)``
- * ``Even more typing in operators (template_fields/ext) (#20608)``
- * ``Use typed Context EVERYWHERE (#20565)``
- * ``Update documentation for November 2021 provider's release (#19882)``
- * ``Prepare documentation for October Provider's release (#19321)``
- * ``More f-strings (#18855)``
- * ``Update documentation for September providers release (#18613)``
- * ``Static start_date and default arg cleanup for misc. provider example DAGs (#18597)``
- * ``Updating miscellaneous provider DAGs to use TaskFlow API where applicable (#18278)``
- * ``Add documentation for January 2021 providers release (#21257)``
-
-2.0.1
-.....
-
-Misc
-~~~~
-
-* ``Optimise connection importing for Airflow 2.2.0``
-
-.. Below changes are excluded from the changelog. Move them to
- appropriate section above if needed. Do not delete the lines(!):
- * ``Update description about the new ''connection-types'' provider meta-data (#17767)``
- * ``Import Hooks lazily individually in providers manager (#17682)``
- * ``Prepares docs for Rc2 release of July providers (#17116)``
- * ``Remove/refactor default_args pattern for miscellaneous providers (#16872)``
- * ``Prepare documentation for July release of providers. (#17015)``
- * ``Removes pylint from our toolchain (#16682)``
-
-2.0.0
-.....
-
-Breaking changes
-~~~~~~~~~~~~~~~~
-
-* ``Auto-apply apply_default decorator (#15667)``
-
-.. warning:: Due to apply_default decorator removal, this version of the provider requires Airflow 2.1.0+.
- If your Airflow version is < 2.1.0, and you want to install this provider version, first upgrade
- Airflow to at least version 2.1.0. Otherwise your Airflow package version will be upgraded
- automatically and you will have to manually run ``airflow upgrade db`` to complete the migration.
-
-Features
-~~~~~~~~
-
-* ``Feature add support for include_headers in get_results (#15598)``
-* ``Feature qubole hook support headers (#15683)``
-* ``Feature qubole hook support headers (#15615)``
-
-.. Below changes are excluded from the changelog. Move them to
- appropriate section above if needed. Do not delete the lines(!):
- * ``Rename the main branch of the Airflow repo to be 'main' (#16149)``
- * ``Prepares provider release after PIP 21 compatibility (#15576)``
- * ``Adds interactivity when generating provider documentation. (#15518)``
- * ``Make Airflow code Pylint 2.8 compatible (#15534)``
- * ``Remove Backport Providers (#14886)``
- * ``Updated documentation for June 2021 provider release (#16294)``
- * ``More documentation update for June providers release (#16405)``
- * ``Synchronizes updated changelog after buggfix release (#16464)``
-
-1.0.2
-.....
-
-Features
-~~~~~~~~
-
-* ``Refactor SQL/BigQuery/Qubole/Druid Check operators (#12677)``
-
-1.0.1
-.....
-
-Updated documentation and readme files.
-
-1.0.0
-.....
-
-Initial version of the provider.
diff --git a/airflow/providers/qubole/__init__.py b/airflow/providers/qubole/__init__.py
deleted file mode 100644
index a03bf8d4e5..0000000000
--- a/airflow/providers/qubole/__init__.py
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
-# OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES.
-#
-# IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE
-# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/provider_packages` DIRECTORY
-#
-from __future__ import annotations
-
-import packaging.version
-
-__all__ = ["__version__"]
-
-__version__ = "3.4.3"
-
-try:
- from airflow import __version__ as airflow_version
-except ImportError:
- from airflow.version import version as airflow_version
-
-if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
- "2.5.0"
-):
- raise RuntimeError(
- f"The package `apache-airflow-providers-qubole:{__version__}` requires Apache Airflow 2.5.0+"
- )
diff --git a/airflow/providers/qubole/hooks/__init__.py b/airflow/providers/qubole/hooks/__init__.py
deleted file mode 100644
index 217e5db960..0000000000
--- a/airflow/providers/qubole/hooks/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/airflow/providers/qubole/hooks/qubole.py b/airflow/providers/qubole/hooks/qubole.py
deleted file mode 100644
index 9e1c535540..0000000000
--- a/airflow/providers/qubole/hooks/qubole.py
+++ /dev/null
@@ -1,304 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""Qubole hook."""
-from __future__ import annotations
-
-import datetime
-import logging
-import os
-import pathlib
-import time
-from typing import TYPE_CHECKING, Any
-
-from qds_sdk.commands import (
- Command,
- DbExportCommand,
- DbImportCommand,
- DbTapQueryCommand,
- HadoopCommand,
- HiveCommand,
- JupyterNotebookCommand,
- PigCommand,
- PrestoCommand,
- ShellCommand,
- SparkCommand,
- SqlCommand,
-)
-from qds_sdk.qubole import Qubole
-
-from airflow.configuration import conf
-from airflow.exceptions import AirflowException
-from airflow.hooks.base import BaseHook
-from airflow.utils.state import State
-
-if TYPE_CHECKING:
- from airflow.models.taskinstance import TaskInstance
- from airflow.serialization.pydantic.taskinstance import TaskInstancePydantic
- from airflow.utils.context import Context
-
-
-log = logging.getLogger(__name__)
-
-COMMAND_CLASSES = {
- "hivecmd": HiveCommand,
- "prestocmd": PrestoCommand,
- "hadoopcmd": HadoopCommand,
- "shellcmd": ShellCommand,
- "pigcmd": PigCommand,
- "sparkcmd": SparkCommand,
- "dbtapquerycmd": DbTapQueryCommand,
- "dbexportcmd": DbExportCommand,
- "dbimportcmd": DbImportCommand,
- "sqlcmd": SqlCommand,
- "jupytercmd": JupyterNotebookCommand,
-}
-
-POSITIONAL_ARGS = {"hadoopcmd": ["sub_command"], "shellcmd": ["parameters"], "pigcmd": ["parameters"]}
-
-
-def flatten_list(list_of_lists) -> list:
- """Flatten the list."""
- return [element for array in list_of_lists for element in array]
-
-
-def filter_options(options: list) -> list:
- """Remove options from the list."""
- options_to_remove = ["help", "print-logs-live", "print-logs", "pool"]
- return [option for option in options if option not in options_to_remove]
-
-
-def get_options_list(command_class) -> list:
- """Get options list."""
- options_list = [option.get_opt_string().strip("--") for option in command_class.optparser.option_list]
- return filter_options(options_list)
-
-
-def build_command_args() -> tuple[dict[str, list], list]:
- """Build Command argument from command and options."""
- command_args, hyphen_args = {}, set()
- for cmd in COMMAND_CLASSES:
- # get all available options from the class
- opts_list = get_options_list(COMMAND_CLASSES[cmd])
-
- # append positional args if any for the command
- if cmd in POSITIONAL_ARGS:
- opts_list += POSITIONAL_ARGS[cmd]
-
- # get args with a hyphen and replace them with underscore
- for index, opt in enumerate(opts_list):
- if "-" in opt:
- opts_list[index] = opt.replace("-", "_")
- hyphen_args.add(opts_list[index])
-
- command_args[cmd] = opts_list
- return command_args, list(hyphen_args)
-
-
-COMMAND_ARGS, HYPHEN_ARGS = build_command_args()
-
-
-class QuboleHook(BaseHook):
- """Hook for Qubole communication."""
-
- conn_name_attr: str = "qubole_conn_id"
- default_conn_name = "qubole_default"
- conn_type = "qubole"
- hook_name = "Qubole"
-
- @staticmethod
- def get_ui_field_behaviour() -> dict[str, Any]:
- """Returns custom field behaviour."""
- return {
- "hidden_fields": ["login", "schema", "port", "extra"],
- "relabeling": {
- "host": "API Endpoint",
- "password": "Auth Token",
- },
- "placeholders": {"host": "https://<env>.qubole.com/api"},
- }
-
- def __init__(self, *args, **kwargs) -> None:
- super().__init__()
- conn = self.get_connection(kwargs.get("qubole_conn_id", self.default_conn_name))
- Qubole.configure(api_token=conn.password, api_url=conn.host)
- self.task_id = kwargs["task_id"]
- self.dag_id = kwargs["dag"].dag_id
- self.kwargs = kwargs
- self.cls = COMMAND_CLASSES[self.kwargs["command_type"]]
- self.cmd: Command | None = None
- self.task_instance: TaskInstance | TaskInstancePydantic | None = None
-
- @staticmethod
- def handle_failure_retry(context) -> None:
- """Handle retries in case of failures."""
- ti = context["ti"]
- cmd_id = ti.xcom_pull(key="qbol_cmd_id", task_ids=ti.task_id)
-
- if cmd_id is not None:
- cmd = Command.find(cmd_id)
- if cmd is not None:
- if cmd.status == "done":
- log.info("Command ID: %s has been succeeded, hence marking this TI as Success.", cmd_id)
- ti.state = State.SUCCESS
- elif cmd.status == "running":
- log.info("Cancelling the Qubole Command Id: %s", cmd_id)
- cmd.cancel()
-
- def execute(self, context: Context) -> None:
- """Execute call."""
- args = self.cls.parse(self.create_cmd_args(context))
- self.cmd = self.cls.create(**args)
- self.task_instance = context["task_instance"]
- context["task_instance"].xcom_push(key="qbol_cmd_id", value=self.cmd.id) # type: ignore[attr-defined]
- self.log.info(
- "Qubole command created with Id: %s and Status: %s",
- self.cmd.id, # type: ignore[attr-defined]
- self.cmd.status, # type: ignore[attr-defined]
- )
-
- while not Command.is_done(self.cmd.status): # type: ignore[attr-defined]
- time.sleep(Qubole.poll_interval)
- self.cmd = self.cls.find(self.cmd.id) # type: ignore[attr-defined]
- self.log.info(
- "Command Id: %s and Status: %s",
- self.cmd.id,
- self.cmd.status, # type: ignore[attr-defined]
- )
-
- if "fetch_logs" in self.kwargs and self.kwargs["fetch_logs"] is True:
- self.log.info(
- "Logs for Command Id: %s \n%s",
- self.cmd.id,
- self.cmd.get_log(), # type: ignore[attr-defined]
- )
-
- if self.cmd.status != "done": # type: ignore[attr-defined]
- raise AirflowException(f"Command Id: {self.cmd.id} failed with Status: {self.cmd.status}")
-
- def kill(self, ti):
- """
- Kill (cancel) a Qubole command.
-
- :param ti: Task Instance of the dag, used to determine the Quboles command id
- :return: response from Qubole
- """
- if self.cmd is None:
- if not ti and not self.task_instance:
- raise Exception("Unable to cancel Qubole Command, context is unavailable!")
- elif not ti:
- ti = self.task_instance
- cmd_id = ti.xcom_pull(key="qbol_cmd_id", task_ids=ti.task_id)
- self.cmd = self.cls.find(cmd_id)
- if self.cls and self.cmd:
- self.log.info("Sending KILL signal to Qubole Command Id: %s", self.cmd.id)
- self.cmd.cancel()
-
- def get_results(
- self,
- ti=None,
- fp=None,
- inline: bool = True,
- delim=None,
- fetch: bool = True,
- include_headers: bool = False,
- ) -> str:
- """
- Get results (or just s3 locations) of a command from Qubole and save into a file.
-
- :param ti: Task Instance of the dag, used to determine the Quboles command id
- :param fp: Optional file pointer, will create one and return if None passed
- :param inline: True to download actual results, False to get s3 locations only
- :param delim: Replaces the CTL-A chars with the given delim, defaults to ','
- :param fetch: when inline is True, get results directly from s3 (if large)
- :return: file location containing actual results or s3 locations of results
- """
- if fp is None:
- iso = datetime.datetime.utcnow().isoformat()
- base_log_folder = conf.get_mandatory_value("logging", "BASE_LOG_FOLDER")
- logpath = os.path.expanduser(base_log_folder)
- resultpath = logpath + "/" + self.dag_id + "/" + self.task_id + "/results"
- pathlib.Path(resultpath).mkdir(parents=True, exist_ok=True)
- fp = open(resultpath + "/" + iso, "wb")
-
- if self.cmd is None:
- cmd_id = ti.xcom_pull(key="qbol_cmd_id", task_ids=self.task_id)
- self.cmd = self.cls.find(cmd_id)
-
- include_headers_str = "true" if include_headers else "false"
- self.cmd.get_results(fp, inline, delim, fetch, arguments=[include_headers_str]) # type: ignore[attr-defined]
- fp.flush()
- fp.close()
- return fp.name
-
- def get_log(self, ti) -> None:
- """
- Get Logs of a command from Qubole.
-
- :param ti: Task Instance of the dag, used to determine the Quboles command id
- :return: command log as text
- """
- if self.cmd is None:
- cmd_id = ti.xcom_pull(key="qbol_cmd_id", task_ids=self.task_id)
- Command.get_log_id(cmd_id)
-
- def get_jobs_id(self, ti) -> None:
- """
- Get jobs associated with a Qubole commands.
-
- :param ti: Task Instance of the dag, used to determine the Quboles command id
- :return: Job information associated with command
- """
- if self.cmd is None:
- cmd_id = ti.xcom_pull(key="qbol_cmd_id", task_ids=self.task_id)
- Command.get_jobs_id(cmd_id)
-
- def create_cmd_args(self, context) -> list[str]:
- """Creates command arguments."""
- args = []
- cmd_type = self.kwargs["command_type"]
- inplace_args = None
- tags = {self.dag_id, self.task_id, context["run_id"]}
- positional_args_list = flatten_list(POSITIONAL_ARGS.values())
-
- for key, value in self.kwargs.items():
- if key in COMMAND_ARGS[cmd_type]:
- if key in HYPHEN_ARGS:
- args.append(f"--{key.replace('_', '-')}={value}")
- elif key in positional_args_list:
- inplace_args = value
- elif key == "tags":
- self._add_tags(tags, value)
- elif key == "notify":
- if value is True:
- args.append("--notify")
- else:
- args.append(f"--{key}={value}")
-
- args.append(f"--tags={','.join(filter(None, tags))}")
-
- if inplace_args is not None:
- args += inplace_args.split(" ")
-
- return args
-
- @staticmethod
- def _add_tags(tags, value) -> None:
- if isinstance(value, str):
- tags.add(value)
- elif isinstance(value, (list, tuple)):
- tags.update(value)
diff --git a/airflow/providers/qubole/hooks/qubole_check.py b/airflow/providers/qubole/hooks/qubole_check.py
deleted file mode 100644
index df94a4c227..0000000000
--- a/airflow/providers/qubole/hooks/qubole_check.py
+++ /dev/null
@@ -1,122 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-import logging
-from io import StringIO
-
-from qds_sdk.commands import Command
-
-from airflow.exceptions import AirflowException
-from airflow.providers.common.sql.hooks.sql import DbApiHook
-from airflow.providers.qubole.hooks.qubole import QuboleHook
-
-log = logging.getLogger(__name__)
-
-COL_DELIM = "\t"
-ROW_DELIM = "\r\n"
-
-
-def isint(value) -> bool:
- """Whether Qubole column are integer."""
- try:
- int(value)
- return True
- except ValueError:
- return False
-
-
-def isfloat(value) -> bool:
- """Whether Qubole column are float."""
- try:
- float(value)
- return True
- except ValueError:
- return False
-
-
-def isbool(value) -> bool:
- """Whether Qubole column are boolean."""
- try:
- return value.lower() in ["true", "false"]
- except ValueError:
- return False
-
-
-def parse_first_row(row_list) -> list[bool | float | int | str]:
- """Parse Qubole first record list."""
- record_list = []
- first_row = row_list[0] if row_list else ""
-
- for col_value in first_row.split(COL_DELIM):
- if isint(col_value):
- col_value = int(col_value)
- elif isfloat(col_value):
- col_value = float(col_value)
- elif isbool(col_value):
- col_value = col_value.lower() == "true"
- record_list.append(col_value)
-
- return record_list
-
-
-class QuboleCheckHook(QuboleHook, DbApiHook):
- """Qubole check hook."""
-
- def __init__(self, context, *args, **kwargs) -> None:
- super().__init__(*args, **kwargs)
- self.results_parser_callable = parse_first_row
- if "results_parser_callable" in kwargs and kwargs["results_parser_callable"] is not None:
- if not callable(kwargs["results_parser_callable"]):
- raise AirflowException("`results_parser_callable` param must be callable")
- self.results_parser_callable = kwargs["results_parser_callable"]
- self.context = context
-
- @staticmethod
- def handle_failure_retry(context) -> None:
- ti = context["ti"]
- cmd_id = ti.xcom_pull(key="qbol_cmd_id", task_ids=ti.task_id)
-
- if cmd_id is not None:
- cmd = Command.find(cmd_id)
- if cmd is not None:
- if cmd.status == "running":
- log.info("Cancelling the Qubole Command Id: %s", cmd_id)
- cmd.cancel()
-
- def get_first(self, sql):
- """Get Qubole query first record list."""
- self.execute(context=self.context)
- query_result = self.get_query_results()
- row_list = list(filter(None, query_result.split(ROW_DELIM)))
- record_list = self.results_parser_callable(row_list)
- return record_list
-
- def get_query_results(self) -> str | None:
- """Get Qubole query result."""
- if self.cmd is not None:
- cmd_id = self.cmd.id
- self.log.info("command id: %d", cmd_id)
- query_result_buffer = StringIO()
- self.cmd.get_results(fp=query_result_buffer, inline=True, delim=COL_DELIM, arguments=["true"])
- query_result = query_result_buffer.getvalue()
- query_result_buffer.close()
- return query_result
- else:
- self.log.error("Qubole command not found")
- return None
diff --git a/airflow/providers/qubole/operators/__init__.py b/airflow/providers/qubole/operators/__init__.py
deleted file mode 100644
index 217e5db960..0000000000
--- a/airflow/providers/qubole/operators/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/airflow/providers/qubole/operators/qubole.py b/airflow/providers/qubole/operators/qubole.py
deleted file mode 100644
index 6869273f56..0000000000
--- a/airflow/providers/qubole/operators/qubole.py
+++ /dev/null
@@ -1,303 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""Qubole operator."""
-from __future__ import annotations
-
-import re
-from typing import TYPE_CHECKING, Sequence
-
-from airflow.hooks.base import BaseHook
-from airflow.models import BaseOperator, BaseOperatorLink, XCom
-from airflow.providers.qubole.hooks.qubole import (
- COMMAND_ARGS,
- HYPHEN_ARGS,
- POSITIONAL_ARGS,
- QuboleHook,
- flatten_list,
-)
-
-if TYPE_CHECKING:
- from airflow.models.taskinstancekey import TaskInstanceKey
- from airflow.utils.context import Context
-
-
-class QDSLink(BaseOperatorLink):
- """Link to QDS."""
-
- name = "Go to QDS"
-
- def get_link(
- self,
- operator: BaseOperator,
- *,
- ti_key: TaskInstanceKey,
- ) -> str:
- """
- Get link to qubole command result page.
-
- :param operator: operator
- :return: url link
- """
- conn = BaseHook.get_connection(
- getattr(operator, "qubole_conn_id", None) or operator.kwargs["qubole_conn_id"] # type: ignore[attr-defined]
- )
- if conn and conn.host:
- host = re.sub(r"api$", "v2/analyze?command_id=", conn.host)
- else:
- host = "https://api.qubole.com/v2/analyze?command_id="
- qds_command_id = XCom.get_value(key="qbol_cmd_id", ti_key=ti_key)
- url = host + str(qds_command_id) if qds_command_id else ""
- return url
-
-
-class QuboleOperator(BaseOperator):
- """
- Execute tasks (commands) on QDS (https://qubole.com).
-
- .. seealso::
- For more information on how to use this operator, take a look at the guide:
- :ref:`howto/operator:QuboleOperator`
-
- :param qubole_conn_id: Connection id which consists of qds auth_token
-
- kwargs:
- :command_type: type of command to be executed, e.g. hivecmd, shellcmd, hadoopcmd
- :tags: array of tags to be assigned with the command
- :cluster_label: cluster label on which the command will be executed
- :name: name to be given to command
- :notify: whether to send email on command completion or not (default is False)
-
- **Arguments specific to command types**
-
- hivecmd:
- :query: inline query statement
- :script_location: s3 location containing query statement
- :sample_size: size of sample in bytes on which to run query
- :macros: macro values which were used in query
- :sample_size: size of sample in bytes on which to run query
- :hive-version: Specifies the hive version to be used. eg: 0.13,1.2,etc.
- prestocmd:
- :query: inline query statement
- :script_location: s3 location containing query statement
- :macros: macro values which were used in query
- hadoopcmd:
- :sub_commnad: must be one these ["jar", "s3distcp", "streaming"] followed by
- 1 or more args
- shellcmd:
- :script: inline command with args
- :script_location: s3 location containing query statement
- :files: list of files in s3 bucket as file1,file2 format. These files will be
- copied into the working directory where the qubole command is being
- executed.
- :archives: list of archives in s3 bucket as archive1,archive2 format. These
- will be unarchived into the working directory where the qubole command is
- being executed
- :parameters: any extra args which need to be passed to script (only when
- script_location is supplied)
- pigcmd:
- :script: inline query statement (latin_statements)
- :script_location: s3 location containing pig query
- :parameters: any extra args which need to be passed to script (only when
- script_location is supplied
- sparkcmd:
- :program: the complete Spark Program in Scala, R, or Python
- :cmdline: spark-submit command line, all required arguments must be specify
- in cmdline itself.
- :sql: inline sql query
- :script_location: s3 location containing query statement
- :language: language of the program, Scala, R, or Python
- :app_id: ID of an Spark job server app
- :arguments: spark-submit command line arguments.
- If `cmdline` is selected, this should not be used because all
- required arguments and configurations are to be passed in the `cmdline` itself.
- :user_program_arguments: arguments that the user program takes in
- :macros: macro values which were used in query
- :note_id: Id of the Notebook to run
- dbtapquerycmd:
- :db_tap_id: data store ID of the target database, in Qubole.
- :query: inline query statement
- :macros: macro values which were used in query
- dbexportcmd:
- :mode: Can be 1 for Hive export or 2 for HDFS/S3 export
- :schema: Db schema name assumed accordingly by database if not specified
- :hive_table: Name of the hive table
- :partition_spec: partition specification for Hive table.
- :dbtap_id: data store ID of the target database, in Qubole.
- :db_table: name of the db table
- :db_update_mode: allowinsert or updateonly
- :db_update_keys: columns used to determine the uniqueness of rows
- :export_dir: HDFS/S3 location from which data will be exported.
- :fields_terminated_by: hex of the char used as column separator in the dataset
- :use_customer_cluster: To use cluster to run command
- :customer_cluster_label: the label of the cluster to run the command on
- :additional_options: Additional Sqoop options which are needed enclose options in
- double or single quotes e.g. '--map-column-hive id=int,data=string'
- dbimportcmd:
- :mode: 1 (simple), 2 (advance)
- :hive_table: Name of the hive table
- :schema: Db schema name assumed accordingly by database if not specified
- :hive_serde: Output format of the Hive Table
- :dbtap_id: data store ID of the target database, in Qubole.
- :db_table: name of the db table
- :where_clause: where clause, if any
- :parallelism: number of parallel db connections to use for extracting data
- :extract_query: SQL query to extract data from db. $CONDITIONS must be part
- of the where clause.
- :boundary_query: Query to be used get range of row IDs to be extracted
- :split_column: Column used as row ID to split data into ranges (mode 2)
- :use_customer_cluster: To use cluster to run command
- :customer_cluster_label: the label of the cluster to run the command on
- :additional_options: Additional Sqoop options which are needed enclose options in
- double or single quotes
- jupytercmd:
- :path: Path including name of the Jupyter notebook to be run with extension.
- :arguments: Valid JSON to be sent to the notebook. Specify the parameters in notebooks and pass
- the parameter value using the JSON format. key is the parameter's name and value is
- the parameter's value. Supported types in parameters are string, integer, float and boolean.
-
- .. note:
-
- Following fields are template-supported : ``query``, ``script_location``,
- ``sub_command``, ``script``, ``files``, ``archives``, ``program``, ``cmdline``,
- ``sql``, ``where_clause``, ``extract_query``, ``boundary_query``, ``macros``,
- ``tags``, ``name``, ``parameters``, ``dbtap_id``, ``hive_table``, ``db_table``,
- ``split_column``, ``note_id``, ``db_update_keys``, ``export_dir``,
- ``partition_spec``, ``qubole_conn_id``, ``arguments``, ``user_program_arguments``.
- You can also use ``.txt`` files for template driven use cases.
-
- .. note:
-
- In QuboleOperator there is a default handler for task failures and retries,
- which generally kills the command running at QDS for the corresponding task
- instance. You can override this behavior by providing your own failure and retry
- handler in task definition.
- """
-
- template_fields: Sequence[str] = (
- "query",
- "script_location",
- "sub_command",
- "script",
- "files",
- "archives",
- "program",
- "cmdline",
- "sql",
- "where_clause",
- "tags",
- "extract_query",
- "boundary_query",
- "macros",
- "name",
- "parameters",
- "dbtap_id",
- "hive_table",
- "db_table",
- "split_column",
- "note_id",
- "db_update_keys",
- "export_dir",
- "partition_spec",
- "qubole_conn_id",
- "arguments",
- "user_program_arguments",
- "cluster_label",
- )
-
- template_ext: Sequence[str] = (".txt",)
- ui_color = "#3064A1"
- ui_fgcolor = "#fff"
- qubole_hook_allowed_args_list = ["command_type", "qubole_conn_id", "fetch_logs"]
-
- operator_extra_links = (QDSLink(),)
-
- def __init__(self, *, qubole_conn_id: str = "qubole_default", **kwargs) -> None:
- self.kwargs = kwargs
- self.kwargs["qubole_conn_id"] = qubole_conn_id
- self.hook: QuboleHook | None = None
- filtered_base_kwargs = self._get_filtered_args(kwargs)
- super().__init__(**filtered_base_kwargs)
-
- if self.on_failure_callback is None:
- self.on_failure_callback = QuboleHook.handle_failure_retry
-
- if self.on_retry_callback is None:
- self.on_retry_callback = QuboleHook.handle_failure_retry
-
- def _get_filtered_args(self, all_kwargs) -> dict:
- qubole_args = (
- flatten_list(COMMAND_ARGS.values())
- + HYPHEN_ARGS
- + flatten_list(POSITIONAL_ARGS.values())
- + self.qubole_hook_allowed_args_list
- )
- return {key: value for key, value in all_kwargs.items() if key not in qubole_args}
-
- def execute(self, context: Context) -> None:
- return self.get_hook().execute(context)
-
- def on_kill(self, ti=None) -> None:
- if self.hook:
- self.hook.kill(ti)
- else:
- self.get_hook().kill(ti)
-
- def get_results(
- self,
- ti=None,
- fp=None,
- inline: bool = True,
- delim=None,
- fetch: bool = True,
- include_headers: bool = False,
- ) -> str:
- """get_results from Qubole."""
- return self.get_hook().get_results(ti, fp, inline, delim, fetch, include_headers)
-
- def get_log(self, ti) -> None:
- """get_log from Qubole."""
- return self.get_hook().get_log(ti)
-
- def get_jobs_id(self, ti) -> None:
- """Get jobs_id from Qubole."""
- return self.get_hook().get_jobs_id(ti)
-
- def get_hook(self) -> QuboleHook:
- """Reinitialising the hook, as some template fields might have changed."""
- return QuboleHook(**self.kwargs)
-
- def __getattribute__(self, name: str) -> str:
- if name in _get_template_fields(self):
- if name in self.kwargs:
- return self.kwargs[name]
- else:
- return ""
- else:
- return object.__getattribute__(self, name)
-
- def __setattr__(self, name: str, value: str) -> None:
- if name in _get_template_fields(self):
- self.kwargs[name] = value
- else:
- object.__setattr__(self, name, value)
-
-
-def _get_template_fields(obj: BaseOperator) -> dict:
- class_ = object.__getattribute__(obj, "__class__")
- template_fields = object.__getattribute__(class_, "template_fields")
- return template_fields
diff --git a/airflow/providers/qubole/operators/qubole_check.py b/airflow/providers/qubole/operators/qubole_check.py
deleted file mode 100644
index 8ecbfc6a57..0000000000
--- a/airflow/providers/qubole/operators/qubole_check.py
+++ /dev/null
@@ -1,210 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-from typing import Callable, Sequence
-
-from airflow.exceptions import AirflowException
-from airflow.providers.common.sql.operators.sql import SQLCheckOperator, SQLValueCheckOperator
-from airflow.providers.qubole.hooks.qubole_check import QuboleCheckHook
-from airflow.providers.qubole.operators.qubole import QuboleOperator
-
-
-class _QuboleCheckOperatorMixin:
- """This is a Mixin for Qubole related check operators."""
-
- kwargs: dict
- results_parser_callable: Callable | None
-
- def execute(self, context=None) -> None:
- """Execute a check operation against Qubole."""
- try:
- self._hook_context = context
- super().execute(context=context) # type: ignore[misc]
- except AirflowException as e:
- handle_airflow_exception(e, self.get_hook())
-
- def get_db_hook(self) -> QuboleCheckHook:
- """Get QuboleCheckHook."""
- return self.get_hook()
-
- def get_hook(self) -> QuboleCheckHook:
- """
- Reinitialising the hook, as some template fields might have changed.
-
- This method overwrites the original QuboleOperator.get_hook() which returns a QuboleHook.
- """
- return QuboleCheckHook(
- context=self._hook_context, results_parser_callable=self.results_parser_callable, **self.kwargs
- )
-
-
-class QuboleCheckOperator(_QuboleCheckOperatorMixin, SQLCheckOperator, QuboleOperator):
- """
- Performs checks against Qubole Commands.
-
- ``QuboleCheckOperator`` expects a command that will be executed on QDS.
- By default, each value on first row of the result of this Qubole Command
- is evaluated using python ``bool`` casting. If any of the
- values return ``False``, the check is failed and errors out.
-
- Note that Python bool casting evals the following as ``False``:
-
- * ``False``
- * ``0``
- * Empty string (``""``)
- * Empty list (``[]``)
- * Empty dictionary or set (``{}``)
-
- Given a query like ``SELECT COUNT(*) FROM foo``, it will fail only if
- the count ``== 0``. You can craft much more complex query that could,
- for instance, check that the table has the same number of rows as
- the source table upstream, or that the count of today's partition is
- greater than yesterday's partition, or that a set of metrics are less
- than 3 standard deviation for the 7 day average.
-
- This operator can be used as a data quality check in your pipeline, and
- depending on where you put it in your DAG, you have the choice to
- stop the critical path, preventing from
- publishing dubious data, or on the side and receive email alerts
- without stopping the progress of the DAG.
-
- .. seealso::
- For more information on how to use this operator, take a look at the guide:
- :ref:`howto/operator:QuboleCheckOperator`
-
- :param qubole_conn_id: Connection id which consists of qds auth_token
- :param results_parser_callable: This is an optional parameter to extend the flexibility of parsing the
- results of Qubole command to the users. This is a Python callable which can hold the logic to parse
- list of rows returned by Qubole command. By default, only the values on first row are used for
- performing checks. This callable should return a list of records on which the checks have to be
- performed.
-
- kwargs:
- Arguments specific to Qubole command can be referred from QuboleOperator docs.
-
- .. note:: All fields in common with template fields of
- QuboleOperator and SQLCheckOperator are template-supported.
-
- """
-
- conn_id_field = "qubole_conn_id"
-
- template_fields: Sequence[str] = tuple(
- set(QuboleOperator.template_fields) | set(SQLCheckOperator.template_fields) | {"qubole_conn_id"}
- )
- template_ext = QuboleOperator.template_ext
- ui_fgcolor = "#000"
-
- def __init__(
- self,
- *,
- qubole_conn_id: str = "qubole_default",
- results_parser_callable: Callable | None = None,
- **kwargs,
- ) -> None:
- sql = get_sql_from_qbol_cmd(kwargs)
- kwargs.pop("sql", None)
- super().__init__(qubole_conn_id=qubole_conn_id, sql=sql, **kwargs)
- self.results_parser_callable = results_parser_callable
- self.on_failure_callback = QuboleCheckHook.handle_failure_retry
- self.on_retry_callback = QuboleCheckHook.handle_failure_retry
- self._hook_context = None
- self.qubole_conn_id = qubole_conn_id
-
-
-# TODO(xinbinhuang): refactor to reduce levels of inheritance
-
-
-class QuboleValueCheckOperator(_QuboleCheckOperatorMixin, SQLValueCheckOperator, QuboleOperator):
- """
- Performs a simple value check using Qubole command.
-
- By default, each value on the first row of this
- Qubole command is compared with a pre-defined value.
- The check fails and errors out if the output of the command
- is not within the permissible limit of expected value.
-
- :param qubole_conn_id: Connection id which consists of qds auth_token
- :param pass_value: Expected value of the query results.
- :param tolerance: Defines the permissible pass_value range, for example if tolerance is 2, the Qubole
- command output can be anything between -2*pass_value and 2*pass_value, without the operator erring
- out.
- :param results_parser_callable: This is an optional parameter to extend the flexibility of parsing the
- results of Qubole command to the users. This is a Python callable which can hold the logic to parse
- list of rows returned by Qubole command. By default, only the values on first row are used for
- performing checks. This callable should return a list of records on which the checks have to be
- performed.
-
- kwargs:
- Arguments specific to Qubole command can be referred from QuboleOperator docs.
-
- .. note:: All fields in common with template fields of
- QuboleOperator and SQLValueCheckOperator are template-supported.
- """
-
- template_fields = tuple(
- set(QuboleOperator.template_fields) | set(SQLValueCheckOperator.template_fields) | {"qubole_conn_id"}
- )
- template_ext = QuboleOperator.template_ext
- ui_fgcolor = "#000"
- conn_id_field = "qubole_conn_id"
-
- def __init__(
- self,
- *,
- pass_value: str | int | float,
- tolerance: int | float | None = None,
- results_parser_callable: Callable | None = None,
- qubole_conn_id: str = "qubole_default",
- **kwargs,
- ) -> None:
- sql = get_sql_from_qbol_cmd(kwargs)
- kwargs.pop("sql", None)
- super().__init__(
- qubole_conn_id=qubole_conn_id, sql=sql, pass_value=pass_value, tolerance=tolerance, **kwargs
- )
- self.results_parser_callable = results_parser_callable
- self.on_failure_callback = QuboleCheckHook.handle_failure_retry
- self.on_retry_callback = QuboleCheckHook.handle_failure_retry
- self._hook_context = None
- self.qubole_conn_id = qubole_conn_id
-
-
-def get_sql_from_qbol_cmd(params) -> str:
- """Get Qubole sql from Qubole command."""
- sql = ""
- if "query" in params:
- sql = params["query"]
- elif "sql" in params:
- sql = params["sql"]
- return sql
-
-
-def handle_airflow_exception(airflow_exception, hook: QuboleCheckHook):
- """Qubole check handle Airflow exception."""
- cmd = hook.cmd
- if cmd is not None:
- if cmd.is_success(cmd.status):
- qubole_command_results = hook.get_query_results()
- qubole_command_id = cmd.id
- exception_message = (
- f"\nQubole Command Id: {qubole_command_id}\nQubole Command Results:\n{qubole_command_results}"
- )
- raise AirflowException(str(airflow_exception) + exception_message)
- raise AirflowException(str(airflow_exception))
diff --git a/airflow/providers/qubole/provider.yaml b/airflow/providers/qubole/provider.yaml
deleted file mode 100644
index 3ffcdbe518..0000000000
--- a/airflow/providers/qubole/provider.yaml
+++ /dev/null
@@ -1,85 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
----
-package-name: apache-airflow-providers-qubole
-name: Qubole
-description: |
- `Qubole <https://www.qubole.com/>`__
-
-suspended: true
-removed: true # TODO: The provider is marked for removal after it's last release is prepared
-
-versions:
- - 3.4.3
- - 3.4.2
- - 3.4.1
- - 3.4.0
- - 3.3.1
- - 3.3.0
- - 3.2.1
- - 3.2.0
- - 3.1.0
- - 3.0.0
- - 2.1.3
- - 2.1.2
- - 2.1.1
- - 2.1.0
- - 2.0.1
- - 2.0.0
- - 1.0.2
- - 1.0.1
- - 1.0.0
-
-dependencies:
- - apache-airflow>=2.5.0
- - apache-airflow-providers-common-sql>=1.3.1
- - qds-sdk>=1.10.4
-
-integrations:
- - integration-name: Qubole
- external-doc-url: https://docs.qubole.com/en/latest/
- how-to-guide:
- - /docs/apache-airflow-providers-qubole/operators/qubole.rst
- - /docs/apache-airflow-providers-qubole/operators/qubole_check.rst
- logo: /integration-logos/qubole/Qubole.png
- tags: [service]
-
-operators:
- - integration-name: Qubole
- python-modules:
- - airflow.providers.qubole.operators.qubole
- - airflow.providers.qubole.operators.qubole_check
-
-sensors:
- - integration-name: Qubole
- python-modules:
- - airflow.providers.qubole.sensors.qubole
-
-hooks:
- - integration-name: Qubole
- python-modules:
- - airflow.providers.qubole.hooks.qubole
- - airflow.providers.qubole.hooks.qubole_check
-
-
-connection-types:
- - hook-class-name: airflow.providers.qubole.hooks.qubole.QuboleHook
- connection-type: qubole
-
-extra-links:
- - airflow.providers.qubole.operators.qubole.QDSLink
diff --git a/airflow/providers/qubole/sensors/__init__.py b/airflow/providers/qubole/sensors/__init__.py
deleted file mode 100644
index 217e5db960..0000000000
--- a/airflow/providers/qubole/sensors/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/airflow/providers/qubole/sensors/qubole.py b/airflow/providers/qubole/sensors/qubole.py
deleted file mode 100644
index 5444f668a2..0000000000
--- a/airflow/providers/qubole/sensors/qubole.py
+++ /dev/null
@@ -1,117 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, Sequence
-
-from qds_sdk.qubole import Qubole
-from qds_sdk.sensors import FileSensor, PartitionSensor
-
-from airflow.exceptions import AirflowException
-from airflow.hooks.base import BaseHook
-from airflow.sensors.base import BaseSensorOperator
-
-if TYPE_CHECKING:
- from airflow.utils.context import Context
-
-
-class QuboleSensor(BaseSensorOperator):
- """Base class for all Qubole Sensors."""
-
- template_fields: Sequence[str] = ("data", "qubole_conn_id")
-
- template_ext: Sequence[str] = (".txt",)
-
- def __init__(self, *, data, qubole_conn_id: str = "qubole_default", **kwargs) -> None:
- self.data = data
- self.qubole_conn_id = qubole_conn_id
-
- if "poke_interval" in kwargs and kwargs["poke_interval"] < 5:
- raise AirflowException(
- f"Sorry, poke_interval can't be less than 5 sec for task '{kwargs['task_id']}' "
- f"in dag '{kwargs['dag'].dag_id}'."
- )
-
- super().__init__(**kwargs)
-
- def poke(self, context: Context) -> bool:
- conn = BaseHook.get_connection(self.qubole_conn_id)
- Qubole.configure(api_token=conn.password, api_url=conn.host)
-
- self.log.info("Poking: %s", self.data)
-
- status = False
- try:
- status = self.sensor_class.check(self.data) # type: ignore[attr-defined]
- except Exception as e:
- self.log.exception(e)
- status = False
-
- self.log.info("Status of this Poke: %s", status)
-
- return status
-
-
-class QuboleFileSensor(QuboleSensor):
- """
- Wait for a file or folder to be present in cloud storage.
-
- Check for file or folder presence via QDS APIs.
-
- .. seealso::
- For more information on how to use this sensor, take a look at the guide:
- :ref:`howto/operator:QuboleFileSensor`
-
- :param qubole_conn_id: Connection id which consists of qds auth_token
- :param data: a JSON object containing payload, whose presence needs to be checked
- Check this `example <https://github.com/apache/airflow/blob/main\
- /airflow/providers/qubole/example_dags/example_qubole_sensor.py>`_ for sample payload
- structure.
-
- .. note:: Both ``data`` and ``qubole_conn_id`` fields support templating. You can
- also use ``.txt`` files for template-driven use cases.
- """
-
- def __init__(self, **kwargs) -> None:
- self.sensor_class = FileSensor
- super().__init__(**kwargs)
-
-
-class QubolePartitionSensor(QuboleSensor):
- """
- Wait for a Hive partition to show up in QHS (Qubole Hive Service).
-
- Check for Hive partition presence via QDS APIs.
-
- .. seealso::
- For more information on how to use this sensor, take a look at the guide:
- :ref:`howto/operator:QubolePartitionSensor`
-
- :param qubole_conn_id: Connection id which consists of qds auth_token
- :param data: a JSON object containing payload, whose presence needs to be checked.
- Check this `example <https://github.com/apache/airflow/blob/main\
- /airflow/providers/qubole/example_dags/example_qubole_sensor.py>`_ for sample payload
- structure.
-
- .. note:: Both ``data`` and ``qubole_conn_id`` fields support templating. You can
- also use ``.txt`` files for template-driven use cases.
- """
-
- def __init__(self, **kwargs) -> None:
- self.sensor_class = PartitionSensor
- super().__init__(**kwargs)
diff --git a/airflow/utils/db.py b/airflow/utils/db.py
index 39f3442997..5b6bd4757e 100644
--- a/airflow/utils/db.py
+++ b/airflow/utils/db.py
@@ -533,14 +533,6 @@ def create_default_connections(session: Session = NEW_SESSION):
),
session,
)
- merge_conn(
- Connection(
- conn_id="qubole_default",
- conn_type="qubole",
- host="localhost",
- ),
- session,
- )
merge_conn(
Connection(
conn_id="redis_default",
diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md
index bb48849e2f..77bc3c4bb9 100644
--- a/dev/README_RELEASE_PROVIDER_PACKAGES.md
+++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md
@@ -136,11 +136,11 @@ breeze release-management prepare-provider-documentation [packages]
NOTE! When you want to release a provider marked for removal (needed in order to prepare last release of the
provider), documentation for the provider will not be prepared when you prepare documentation for
all providers - you have to specifically use the provider name in a separate command.
-For example to prepare documentation for `qubole` provider marked for removal you need to run
+For example to prepare documentation for `removed.provider` provider marked for removal you need to run
separately this command:
```shell script
-breeze release-management prepare-provider-documentation qubole
+breeze release-management prepare-provider-documentation removed.provider
```
This command will not only prepare documentation but will also help the release manager to review
@@ -240,11 +240,11 @@ breeze release-management prepare-provider-packages --package-format both PACKAG
NOTE! When you want to release a provider marked for removal (needed in order to prepare last release of the
provider), package for the provider will not be prepared when you prepare documentation for
all providers - you have to specifically use the provider name in a separate command.
-For example to prepare documentation for `qubole` provider marked for removal you need to run
+For example to prepare documentation for `removed.provider` provider marked for removal you need to run
separately this command:
```shell script
-breeze release-management prepare-provider-packages --package-format both qubole
+breeze release-management prepare-provider-packages --package-format both removed.provider
```
@@ -320,11 +320,11 @@ breeze release-management prepare-provider-packages --version-suffix-for-pypi rc
NOTE! When you want to release a provider marked for removal (needed in order to prepare last release of the
provider), package for the provider will not be prepared when you prepare documentation for
all providers - you have to specifically use the provider name in a separate command.
-For example to prepare documentation for `qubole` provider marked for removal you need to run
+For example to prepare documentation for `removed.provider` provider marked for removal you need to run
separately this command:
```shell script
-breeze release-management prepare-provider-packages --package-format both qubole
+breeze release-management prepare-provider-packages --package-format both removed.provider
```
@@ -425,11 +425,11 @@ breeze build-docs apache-airflow-providers cncf.kubernetes sftp --clean-build
NOTE! When you want to release a provider marked for removal (needed in order to prepare last release of the
provider), doc for the provider will not be built when you prepare documentation for
all providers - you have to specifically use the provider name in a separate command.
-For example to prepare documentation for `qubole` provider marked for removal you need to run
+For example to prepare documentation for `removed.provider` provider marked for removal you need to run
separately this command:
```shell script
-breeze build-docs qubole
+breeze build-docs removed.provider
```
@@ -472,11 +472,11 @@ breeze release-management add-back-references all-providers
NOTE! When you want to release a provider marked for removal (needed in order to prepare last release of the
provider), docs for the provider will not be published when you prepare documentation for
all providers - you have to specifically use the provider name in a separate command.
-For example to prepare documentation for `qubole` provider marked for removal you need to run
+For example to prepare documentation for `removed.provider` provider marked for removal you need to run
separately this command:
```shell script
-breeze release-management publish-docs qubole
+breeze release-management publish-docs removed.provider
breeze release-management add-back-references all-providers
```
diff --git a/docs/apache-airflow-providers-qubole/changelog.rst b/docs/apache-airflow-providers-qubole/changelog.rst
deleted file mode 100644
index 8e1f770b1b..0000000000
--- a/docs/apache-airflow-providers-qubole/changelog.rst
+++ /dev/null
@@ -1,18 +0,0 @@
-.. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-.. include:: ../../airflow/providers/qubole/CHANGELOG.rst
diff --git a/docs/apache-airflow-providers-qubole/commits.rst b/docs/apache-airflow-providers-qubole/commits.rst
deleted file mode 100644
index 7138316028..0000000000
--- a/docs/apache-airflow-providers-qubole/commits.rst
+++ /dev/null
@@ -1,390 +0,0 @@
-.. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME!
-
-Package apache-airflow-providers-qubole
-------------------------------------------------------
-
-`Qubole <https://www.qubole.com/>`__
-
-
-This is detailed commit list of changes for versions provider package: ``qubole``.
-For high-level changelog, see :doc:`package information including changelog <index>`.
-
-
-
-3.4.3
-.....
-
-Latest change: 2023-11-07
-
-================================================================================================= =========== ====================================================================================
-Commit Committed Subject
-================================================================================================= =========== ====================================================================================
-`3b8db612ff <https://github.com/apache/airflow/commit/3b8db612ff39abbf9e965081c859e9e439ed832d>`_ 2023-11-07 ``Add information about Qubole removal and make it possible to release it (#35492)``
-`63cc915cd3 <https://github.com/apache/airflow/commit/63cc915cd38a5034df6bf9c618e12f8690eeade0>`_ 2023-10-31 ``Switch from Black to Ruff formatter (#35287)``
-`dd7ba3cae1 <https://github.com/apache/airflow/commit/dd7ba3cae139cb10d71c5ebc25fc496c67ee784e>`_ 2023-10-19 ``Pre-upgrade 'ruff==0.0.292' changes in providers (#35053)``
-`b75f9e8806 <https://github.com/apache/airflow/commit/b75f9e880614fa0427e7d24a1817955f5de658b3>`_ 2023-10-18 ``Upgrade pre-commits (#35033)``
-`e9987d5059 <https://github.com/apache/airflow/commit/e9987d50598f70d84cbb2a5d964e21020e81c080>`_ 2023-10-13 ``Prepare docs 1st wave of Providers in October 2023 (#34916)``
-`0c8e30e43b <https://github.com/apache/airflow/commit/0c8e30e43b70e9d033e1686b327eb00aab82479c>`_ 2023-10-05 ``Bump min airflow version of providers (#34728)``
-`1f0e673ce3 <https://github.com/apache/airflow/commit/1f0e673ce3db5925dc1075e921ae204aa653118a>`_ 2023-08-29 ``Suspend qubole provider (#33889)``
-================================================================================================= =========== ====================================================================================
-
-3.4.2
-.....
-
-Latest change: 2023-08-11
-
-================================================================================================= =========== ==========================================================================
-Commit Committed Subject
-================================================================================================= =========== ==========================================================================
-`b5a4d36383 <https://github.com/apache/airflow/commit/b5a4d36383c4143f46e168b8b7a4ba2dc7c54076>`_ 2023-08-11 ``Prepare docs for Aug 2023 2nd wave of Providers (#33291)``
-`9736143468 <https://github.com/apache/airflow/commit/9736143468cfe034e65afb3df3031ab3626f0f6d>`_ 2023-08-07 ``Add a new parameter to SQL operators to specify conn id field (#30784)``
-`6b892bf21f <https://github.com/apache/airflow/commit/6b892bf21f69a41caf670e2c498aea1c83086848>`_ 2023-08-06 ``Upgrade ruff to latest 0.0.282 version in pre-commits (#33152)``
-`225e3041d2 <https://github.com/apache/airflow/commit/225e3041d269698d0456e09586924c1898d09434>`_ 2023-07-06 ``Prepare docs for July 2023 wave of Providers (RC2) (#32381)``
-`e45bee8840 <https://github.com/apache/airflow/commit/e45bee884068399e7265421511e17fed106ce5b4>`_ 2023-07-05 ``D205 Support - Providers: Pagerduty to SMTP (inclusive) (#32358)``
-`3878fe6fab <https://github.com/apache/airflow/commit/3878fe6fab3ccc1461932b456c48996f2763139f>`_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)``
-`cb4927a018 <https://github.com/apache/airflow/commit/cb4927a01887e2413c45d8d9cb63e74aa994ee74>`_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)``
-`09d4718d3a <https://github.com/apache/airflow/commit/09d4718d3a46aecf3355d14d3d23022002f4a818>`_ 2023-06-27 ``Improve provider documentation and README structure (#32125)``
-================================================================================================= =========== ==========================================================================
-
-3.4.1
-.....
-
-Latest change: 2023-06-20
-
-================================================================================================= =========== =============================================================
-Commit Committed Subject
-================================================================================================= =========== =============================================================
-`79bcc2e668 <https://github.com/apache/airflow/commit/79bcc2e668e648098aad6eaa87fe8823c76bc69a>`_ 2023-06-20 ``Prepare RC1 docs for June 2023 wave of Providers (#32001)``
-`8b146152d6 <https://github.com/apache/airflow/commit/8b146152d62118defb3004c997c89c99348ef948>`_ 2023-06-20 ``Add note about dropping Python 3.7 for providers (#32015)``
-`a59076eaee <https://github.com/apache/airflow/commit/a59076eaeed03dd46e749ad58160193b4ef3660c>`_ 2023-06-02 ``Add D400 pydocstyle check - Providers (#31427)``
-================================================================================================= =========== =============================================================
-
-3.4.0
-.....
-
-Latest change: 2023-05-19
-
-================================================================================================= =========== ======================================================================================
-Commit Committed Subject
-================================================================================================= =========== ======================================================================================
-`45548b9451 <https://github.com/apache/airflow/commit/45548b9451fba4e48c6f0c0ba6050482c2ea2956>`_ 2023-05-19 ``Prepare RC2 docs for May 2023 wave of Providers (#31416)``
-`abea189022 <https://github.com/apache/airflow/commit/abea18902257c0250fedb764edda462f9e5abc84>`_ 2023-05-18 ``Use '__version__' in providers not 'version' (#31393)``
-`f5aed58d9f <https://github.com/apache/airflow/commit/f5aed58d9fb2137fa5f0e3ce75b6709bf8393a94>`_ 2023-05-18 ``Fixing circular import error in providers caused by airflow version check (#31379)``
-`d9ff55cf6d <https://github.com/apache/airflow/commit/d9ff55cf6d95bb342fed7a87613db7b9e7c8dd0f>`_ 2023-05-16 ``Prepare docs for May 2023 wave of Providers (#31252)``
-`ac46902154 <https://github.com/apache/airflow/commit/ac46902154c060246dec942f921f7670015e6031>`_ 2023-05-04 ``Move TaskInstanceKey to a separate file (#31033)``
-`eef5bc7f16 <https://github.com/apache/airflow/commit/eef5bc7f166dc357fea0cc592d39714b1a5e3c14>`_ 2023-05-03 ``Add full automation for min Airflow version for providers (#30994)``
-`a7eb32a5b2 <https://github.com/apache/airflow/commit/a7eb32a5b222e236454d3e474eec478ded7c368d>`_ 2023-04-30 ``Bump minimum Airflow version in providers (#30917)``
-`58294c5440 <https://github.com/apache/airflow/commit/58294c5440608b1a58828cbae36f91b7148c04b4>`_ 2023-04-10 ``Decouple "job runner" from BaseJob ORM model (#30255)``
-`d23a3bbed8 <https://github.com/apache/airflow/commit/d23a3bbed89ae04369983f21455bf85ccc1ae1cb>`_ 2023-04-04 ``Add mechanism to suspend providers (#30422)``
-================================================================================================= =========== ======================================================================================
-
-3.3.1
-.....
-
-Latest change: 2022-11-26
-
-================================================================================================= =========== ================================================================
-Commit Committed Subject
-================================================================================================= =========== ================================================================
-`25bdbc8e67 <https://github.com/apache/airflow/commit/25bdbc8e6768712bad6043618242eec9c6632618>`_ 2022-11-26 ``Updated docs for RC3 wave of providers (#27937)``
-`2e20e9f7eb <https://github.com/apache/airflow/commit/2e20e9f7ebf5f43bf27069f4c0063cdd72e6b2e2>`_ 2022-11-24 ``Prepare for follow-up relase for November providers (#27774)``
-`80c327bd3b <https://github.com/apache/airflow/commit/80c327bd3b45807ff2e38d532325bccd6fe0ede0>`_ 2022-11-24 ``Bump common.sql provider to 1.3.1 (#27888)``
-================================================================================================= =========== ================================================================
-
-3.3.0
-.....
-
-Latest change: 2022-11-15
-
-================================================================================================= =========== =========================================================================
-Commit Committed Subject
-================================================================================================= =========== =========================================================================
-`12c3c39d1a <https://github.com/apache/airflow/commit/12c3c39d1a816c99c626fe4c650e88cf7b1cc1bc>`_ 2022-11-15 ``pRepare docs for November 2022 wave of Providers (#27613)``
-`9ab1a6a3e7 <https://github.com/apache/airflow/commit/9ab1a6a3e70b32a3cddddf0adede5d2f3f7e29ea>`_ 2022-10-27 ``Update old style typing (#26872)``
-`78b8ea2f22 <https://github.com/apache/airflow/commit/78b8ea2f22239db3ef9976301234a66e50b47a94>`_ 2022-10-24 ``Move min airflow version to 2.3.0 for all providers (#27196)``
-`2a34dc9e84 <https://github.com/apache/airflow/commit/2a34dc9e8470285b0ed2db71109ef4265e29688b>`_ 2022-10-23 ``Enable string normalization in python formatting - providers (#27205)``
-================================================================================================= =========== =========================================================================
-
-3.2.1
-.....
-
-Latest change: 2022-09-28
-
-================================================================================================= =========== ====================================================================================
-Commit Committed Subject
-================================================================================================= =========== ====================================================================================
-`f8db64c35c <https://github.com/apache/airflow/commit/f8db64c35c8589840591021a48901577cff39c07>`_ 2022-09-28 ``Update docs for September Provider's release (#26731)``
-`06acf40a43 <https://github.com/apache/airflow/commit/06acf40a4337759797f666d5bb27a5a393b74fed>`_ 2022-09-13 ``Apply PEP-563 (Postponed Evaluation of Annotations) to non-core airflow (#26289)``
-`5066844513 <https://github.com/apache/airflow/commit/50668445137e4037bb4a3b652bec22e53d1eddd7>`_ 2022-09-09 ``D400 first line should end with period batch02 (#25268)``
-================================================================================================= =========== ====================================================================================
-
-3.2.0
-.....
-
-Latest change: 2022-08-10
-
-================================================================================================= =========== ============================================================================
-Commit Committed Subject
-================================================================================================= =========== ============================================================================
-`e5ac6c7cfb <https://github.com/apache/airflow/commit/e5ac6c7cfb189c33e3b247f7d5aec59fe5e89a00>`_ 2022-08-10 ``Prepare docs for new providers release (August 2022) (#25618)``
-`d9e924c058 <https://github.com/apache/airflow/commit/d9e924c058f5da9eba5bb5b85a04bfea6fb2471a>`_ 2022-08-05 ``Make extra link work in UI (#25500)``
-`44a1cdc1e1 <https://github.com/apache/airflow/commit/44a1cdc1e15c818d5c899dedf2b4367e4a97df20>`_ 2022-08-04 ``Correctly render 'results_parser_callable' param in Qubole docs (#25514)``
-`acab8f52dd <https://github.com/apache/airflow/commit/acab8f52dd8d90fd6583779127895dd343780f79>`_ 2022-07-29 ``Move all "old" SQL operators to common.sql providers (#25350)``
-`c8af0592c0 <https://github.com/apache/airflow/commit/c8af0592c08017ee48f69f608ad4a6529ee14292>`_ 2022-07-26 ``Improve taskflow type hints with ParamSpec (#25173)``
-================================================================================================= =========== ============================================================================
-
-3.1.0
-.....
-
-Latest change: 2022-07-13
-
-================================================================================================= =========== ==================================================================
-Commit Committed Subject
-================================================================================================= =========== ==================================================================
-`d2459a241b <https://github.com/apache/airflow/commit/d2459a241b54d596ebdb9d81637400279fff4f2d>`_ 2022-07-13 ``Add documentation for July 2022 Provider's release (#25030)``
-`46bbfdade0 <https://github.com/apache/airflow/commit/46bbfdade0638cb8a5d187e47034b84e68ddf762>`_ 2022-07-07 ``Move all SQL classes to common-sql provider (#24836)``
-`96b01a8012 <https://github.com/apache/airflow/commit/96b01a8012d164df7c24c460149d3b79ecad3901>`_ 2022-07-05 ``Remove "bad characters" from our codebase (#24841)``
-`0de31bd73a <https://github.com/apache/airflow/commit/0de31bd73a8f41dded2907f0dee59dfa6c1ed7a1>`_ 2022-06-29 ``Move provider dependencies to inside provider folders (#24672)``
-`510a6bab45 <https://github.com/apache/airflow/commit/510a6bab4595cce8bd5b1447db957309d70f35d9>`_ 2022-06-28 ``Remove 'hook-class-names' from provider.yaml (#24702)``
-================================================================================================= =========== ==================================================================
-
-3.0.0
-.....
-
-Latest change: 2022-06-09
-
-================================================================================================= =========== ==================================================================================
-Commit Committed Subject
-================================================================================================= =========== ==================================================================================
-`dcdcf3a2b8 <https://github.com/apache/airflow/commit/dcdcf3a2b8054fa727efb4cd79d38d2c9c7e1bd5>`_ 2022-06-09 ``Update release notes for RC2 release of Providers for May 2022 (#24307)``
-`717a7588bc <https://github.com/apache/airflow/commit/717a7588bc8170363fea5cb75f17efcf68689619>`_ 2022-06-07 ``Update package description to remove double min-airflow specification (#24292)``
-`aeabe994b3 <https://github.com/apache/airflow/commit/aeabe994b3381d082f75678a159ddbb3cbf6f4d3>`_ 2022-06-07 ``Prepare docs for May 2022 provider's release (#24231)``
-`9dc2851671 <https://github.com/apache/airflow/commit/9dc2851671cd5cdce445f01f380985f2d7a9b4cf>`_ 2022-06-05 ``Fix backwards-compatibility introduced by fixing mypy problems (#24230)``
-`027b707d21 <https://github.com/apache/airflow/commit/027b707d215a9ff1151717439790effd44bab508>`_ 2022-06-05 ``Add explanatory note for contributors about updating Changelog (#24229)``
-`e3824ce521 <https://github.com/apache/airflow/commit/e3824ce52181089779a409e5ff64fbf9677cccfc>`_ 2022-06-03 ``Migrate Qubole example DAGs to new design #22460 (#24149)``
-`71e4deb1b0 <https://github.com/apache/airflow/commit/71e4deb1b093b7ad9320eb5eb34eca8ea440a238>`_ 2022-05-16 ``Add typing for airflow/configuration.py (#23716)``
-`75c60923e0 <https://github.com/apache/airflow/commit/75c60923e01375ffc5f71c4f2f7968f489e2ca2f>`_ 2022-05-12 ``Prepare provider documentation 2022.05.11 (#23631)``
-`8b6b0848a3 <https://github.com/apache/airflow/commit/8b6b0848a3cacf9999477d6af4d2a87463f03026>`_ 2022-04-23 ``Use new Breese for building, pulling and verifying the images. (#23104)``
-`49e336ae03 <https://github.com/apache/airflow/commit/49e336ae0302b386a2f47269a6d13988382d975f>`_ 2022-04-13 ``Replace usage of 'DummyOperator' with 'EmptyOperator' (#22974)``
-================================================================================================= =========== ==================================================================================
-
-2.1.3
-.....
-
-Latest change: 2022-03-22
-
-================================================================================================= =========== ==============================================================
-Commit Committed Subject
-================================================================================================= =========== ==============================================================
-`d7dbfb7e26 <https://github.com/apache/airflow/commit/d7dbfb7e26a50130d3550e781dc71a5fbcaeb3d2>`_ 2022-03-22 ``Add documentation for bugfix release of Providers (#22383)``
-================================================================================================= =========== ==============================================================
-
-2.1.2
-.....
-
-Latest change: 2022-03-14
-
-================================================================================================= =========== ====================================================================
-Commit Committed Subject
-================================================================================================= =========== ====================================================================
-`16adc035b1 <https://github.com/apache/airflow/commit/16adc035b1ecdf533f44fbb3e32bea972127bb71>`_ 2022-03-14 ``Add documentation for Classifier release for March 2022 (#22226)``
-`c1ab8e2d7b <https://github.com/apache/airflow/commit/c1ab8e2d7b68a31408e750129592e16432474512>`_ 2022-03-14 ``Protect against accidental misuse of XCom.get_value() (#22244)``
-`d08284ed25 <https://github.com/apache/airflow/commit/d08284ed251b7c5712190181623b500a38cd640d>`_ 2022-03-11 `` Add map_index to XCom model and interface (#22112)``
-================================================================================================= =========== ====================================================================
-
-2.1.1
-.....
-
-Latest change: 2022-03-07
-
-================================================================================================= =========== ===============================================================================
-Commit Committed Subject
-================================================================================================= =========== ===============================================================================
-`f5b96315fe <https://github.com/apache/airflow/commit/f5b96315fe65b99c0e2542831ff73a3406c4232d>`_ 2022-03-07 ``Add documentation for Feb Providers release (#22056)``
-`08575ddd8a <https://github.com/apache/airflow/commit/08575ddd8a72f96a3439f73e973ee9958188eb83>`_ 2022-03-01 ``Change BaseOperatorLink interface to take a ti_key, not a datetime (#21798)``
-================================================================================================= =========== ===============================================================================
-
-2.1.0
-.....
-
-Latest change: 2022-02-08
-
-================================================================================================= =========== ======================================================================================
-Commit Committed Subject
-================================================================================================= =========== ======================================================================================
-`d94fa37830 <https://github.com/apache/airflow/commit/d94fa378305957358b910cfb1fe7cb14bc793804>`_ 2022-02-08 ``Fixed changelog for January 2022 (delayed) provider's release (#21439)``
-`6c3a67d4fc <https://github.com/apache/airflow/commit/6c3a67d4fccafe4ab6cd9ec8c7bacf2677f17038>`_ 2022-02-05 ``Add documentation for January 2021 providers release (#21257)``
-`ddb5246bd1 <https://github.com/apache/airflow/commit/ddb5246bd1576e2ce6abf8c80c3328d7d71a75ce>`_ 2022-02-03 ``Refactor operator links to not create ad hoc TaskInstances (#21285)``
-`cb73053211 <https://github.com/apache/airflow/commit/cb73053211367e2c2dd76d5279cdc7dc7b190124>`_ 2022-01-27 ``Add optional features in providers. (#21074)``
-`602abe8394 <https://github.com/apache/airflow/commit/602abe8394fafe7de54df7e73af56de848cdf617>`_ 2022-01-20 ``Remove ':type' lines now sphinx-autoapi supports typehints (#20951)``
-`9f135bce30 <https://github.com/apache/airflow/commit/9f135bce307a527e38f81cdf8100e0b9d1a53929>`_ 2022-01-04 ``Fxing MyPy issues inside airflow/providers/qubole (#20625)``
-`f77417eb0d <https://github.com/apache/airflow/commit/f77417eb0d3f12e4849d80645325c02a48829278>`_ 2021-12-31 ``Fix K8S changelog to be PyPI-compatible (#20614)``
-`97496ba2b4 <https://github.com/apache/airflow/commit/97496ba2b41063fa24393c58c5c648a0cdb5a7f8>`_ 2021-12-31 ``Update documentation for provider December 2021 release (#20523)``
-`83f8e178ba <https://github.com/apache/airflow/commit/83f8e178ba7a3d4ca012c831a5bfc2cade9e812d>`_ 2021-12-31 ``Even more typing in operators (template_fields/ext) (#20608)``
-`d56e7b56bb <https://github.com/apache/airflow/commit/d56e7b56bb9827daaf8890557147fd10bdf72a7e>`_ 2021-12-30 ``Fix template_fields type to have MyPy friendly Sequence type (#20571)``
-`a0821235fb <https://github.com/apache/airflow/commit/a0821235fb6877a471973295fe42283ef452abf6>`_ 2021-12-30 ``Use typed Context EVERYWHERE (#20565)``
-`7d84196519 <https://github.com/apache/airflow/commit/7d84196519fcdbf96204d754d95c4dbca1ba9121>`_ 2021-12-19 ``Fix MyPy Errors for Qubole provider. (#20319)``
-`2fbfbef17b <https://github.com/apache/airflow/commit/2fbfbef17b2f3c683a9d9de8ced190a13f06712a>`_ 2021-12-06 ``Add Qubole how to documentation (#20058)``
-`853576d901 <https://github.com/apache/airflow/commit/853576d9019d2aca8de1d9c587c883dcbe95b46a>`_ 2021-11-30 ``Update documentation for November 2021 provider's release (#19882)``
-`d9567eb106 <https://github.com/apache/airflow/commit/d9567eb106929b21329c01171fd398fbef2dc6c6>`_ 2021-10-29 ``Prepare documentation for October Provider's release (#19321)``
-`86a2a19ad2 <https://github.com/apache/airflow/commit/86a2a19ad2bdc87a9ad14bb7fde9313b2d7489bb>`_ 2021-10-17 ``More f-strings (#18855)``
-`840ea3efb9 <https://github.com/apache/airflow/commit/840ea3efb9533837e9f36b75fa527a0fbafeb23a>`_ 2021-09-30 ``Update documentation for September providers release (#18613)``
-`ef037e7021 <https://github.com/apache/airflow/commit/ef037e702182e4370cb00c853c4fb0e246a0479c>`_ 2021-09-29 ``Static start_date and default arg cleanup for misc. provider example DAGs (#18597)``
-`a458fcc573 <https://github.com/apache/airflow/commit/a458fcc573845ff65244a2dafd204ed70129f3e8>`_ 2021-09-27 ``Updating miscellaneous provider DAGs to use TaskFlow API where applicable (#18278)``
-================================================================================================= =========== ======================================================================================
-
-2.0.1
-.....
-
-Latest change: 2021-08-30
-
-================================================================================================= =========== =============================================================================
-Commit Committed Subject
-================================================================================================= =========== =============================================================================
-`0a68588479 <https://github.com/apache/airflow/commit/0a68588479e34cf175d744ea77b283d9d78ea71a>`_ 2021-08-30 ``Add August 2021 Provider's documentation (#17890)``
-`be75dcd39c <https://github.com/apache/airflow/commit/be75dcd39cd10264048c86e74110365bd5daf8b7>`_ 2021-08-23 ``Update description about the new ''connection-types'' provider meta-data``
-`76ed2a49c6 <https://github.com/apache/airflow/commit/76ed2a49c6cd285bf59706cf04f39a7444c382c9>`_ 2021-08-19 ``Import Hooks lazily individually in providers manager (#17682)``
-`87f408b1e7 <https://github.com/apache/airflow/commit/87f408b1e78968580c760acb275ae5bb042161db>`_ 2021-07-26 ``Prepares docs for Rc2 release of July providers (#17116)``
-`0dbd0f420c <https://github.com/apache/airflow/commit/0dbd0f420cc08e011317e2a9f21f92fff4a64c1b>`_ 2021-07-26 ``Remove/refactor default_args pattern for miscellaneous providers (#16872)``
-`b916b75079 <https://github.com/apache/airflow/commit/b916b7507921129dc48d6add1bdc4b923b60c9b9>`_ 2021-07-15 ``Prepare documentation for July release of providers. (#17015)``
-`866a601b76 <https://github.com/apache/airflow/commit/866a601b76e219b3c043e1dbbc8fb22300866351>`_ 2021-06-28 ``Removes pylint from our toolchain (#16682)``
-================================================================================================= =========== =============================================================================
-
-2.0.0
-.....
-
-Latest change: 2021-06-18
-
-================================================================================================= =========== =======================================================================
-Commit Committed Subject
-================================================================================================= =========== =======================================================================
-`bbc627a3da <https://github.com/apache/airflow/commit/bbc627a3dab17ba4cf920dd1a26dbed6f5cebfd1>`_ 2021-06-18 ``Prepares documentation for rc2 release of Providers (#16501)``
-`cbf8001d76 <https://github.com/apache/airflow/commit/cbf8001d7630530773f623a786f9eb319783b33c>`_ 2021-06-16 ``Synchronizes updated changelog after buggfix release (#16464)``
-`1fba5402bb <https://github.com/apache/airflow/commit/1fba5402bb14b3ffa6429fdc683121935f88472f>`_ 2021-06-15 ``More documentation update for June providers release (#16405)``
-`9c94b72d44 <https://github.com/apache/airflow/commit/9c94b72d440b18a9e42123d20d48b951712038f9>`_ 2021-06-07 ``Updated documentation for June 2021 provider release (#16294)``
-`1e647029e4 <https://github.com/apache/airflow/commit/1e647029e469c1bb17e9ad051d0184f3357644c3>`_ 2021-06-01 ``Rename the main branch of the Airflow repo to be 'main' (#16149)``
-`996965aad9 <https://github.com/apache/airflow/commit/996965aad9874e9c6dad0a1f147d779adc462278>`_ 2021-05-11 ``Feature qubole hook support headers (#15683)``
-`37681bca00 <https://github.com/apache/airflow/commit/37681bca0081dd228ac4047c17631867bba7a66f>`_ 2021-05-07 ``Auto-apply apply_default decorator (#15667)``
-`47a5539f7b <https://github.com/apache/airflow/commit/47a5539f7b83826b85b189b58b1641798d637369>`_ 2021-05-04 ``Feature qubole hook support headers (#15615)``
-`807ad32ce5 <https://github.com/apache/airflow/commit/807ad32ce59e001cb3532d98a05fa7d0d7fabb95>`_ 2021-05-01 ``Prepares provider release after PIP 21 compatibility (#15576)``
-`40a2476a5d <https://github.com/apache/airflow/commit/40a2476a5db14ee26b5108d72635da116eab720b>`_ 2021-04-28 ``Adds interactivity when generating provider documentation. (#15518)``
-`4b031d39e1 <https://github.com/apache/airflow/commit/4b031d39e12110f337151cda6693e2541bf71c2c>`_ 2021-04-27 ``Make Airflow code Pylint 2.8 compatible (#15534)``
-`68e4c4dcb0 <https://github.com/apache/airflow/commit/68e4c4dcb0416eb51a7011a3bb040f1e23d7bba8>`_ 2021-03-20 ``Remove Backport Providers (#14886)``
-================================================================================================= =========== =======================================================================
-
-1.0.2
-.....
-
-Latest change: 2021-02-27
-
-================================================================================================= =========== ===============================================================
-Commit Committed Subject
-================================================================================================= =========== ===============================================================
-`589d6dec92 <https://github.com/apache/airflow/commit/589d6dec922565897785bcbc5ac6bb3b973d7f5d>`_ 2021-02-27 ``Prepare to release the next wave of providers: (#14487)``
-`33214d9326 <https://github.com/apache/airflow/commit/33214d9326bb0bb52f06e230895f4f68fc952664>`_ 2021-02-26 ``Refactor SQL/BigQuery/Qubole/Druid Check operators (#12677)``
-================================================================================================= =========== ===============================================================
-
-1.0.1
-.....
-
-Latest change: 2021-02-04
-
-================================================================================================= =========== ========================================================
-Commit Committed Subject
-================================================================================================= =========== ========================================================
-`88bdcfa0df <https://github.com/apache/airflow/commit/88bdcfa0df5bcb4c489486e05826544b428c8f43>`_ 2021-02-04 ``Prepare to release a new wave of providers. (#14013)``
-`ac2f72c98d <https://github.com/apache/airflow/commit/ac2f72c98dc0821b33721054588adbf2bb53bb0b>`_ 2021-02-01 ``Implement provider versioning tools (#13767)``
-`a9ac2b040b <https://github.com/apache/airflow/commit/a9ac2b040b64de1aa5d9c2b9def33334e36a8d22>`_ 2021-01-23 ``Switch to f-strings using flynt. (#13732)``
-`3fd5ef3555 <https://github.com/apache/airflow/commit/3fd5ef355556cf0ad7896bb570bbe4b2eabbf46e>`_ 2021-01-21 ``Add missing logos for integrations (#13717)``
-`295d66f914 <https://github.com/apache/airflow/commit/295d66f91446a69610576d040ba687b38f1c5d0a>`_ 2020-12-30 ``Fix Grammar in PIP warning (#13380)``
-`6cf76d7ac0 <https://github.com/apache/airflow/commit/6cf76d7ac01270930de7f105fb26428763ee1d4e>`_ 2020-12-18 ``Fix typo in pip upgrade command :( (#13148)``
-================================================================================================= =========== ========================================================
-
-1.0.0
-.....
-
-Latest change: 2020-12-09
-
-================================================================================================= =========== ======================================================================================================================================================================
-Commit Committed Subject
-================================================================================================= =========== ======================================================================================================================================================================
-`32971a1a2d <https://github.com/apache/airflow/commit/32971a1a2de1db0b4f7442ed26facdf8d3b7a36f>`_ 2020-12-09 ``Updates providers versions to 1.0.0 (#12955)``
-`d5589673a9 <https://github.com/apache/airflow/commit/d5589673a95aaced0b851ea0a4061a010a924a82>`_ 2020-12-08 ``Move dummy_operator.py to dummy.py (#11178) (#11293)``
-`b40dffa085 <https://github.com/apache/airflow/commit/b40dffa08547b610162f8cacfa75847f3c4ca364>`_ 2020-12-08 ``Rename remaing modules to match AIP-21 (#12917)``
-`9b39f24780 <https://github.com/apache/airflow/commit/9b39f24780e85f859236672e9060b2fbeee81b36>`_ 2020-12-08 ``Add support for dynamic connection form fields per provider (#12558)``
-`1dcd3e13fd <https://github.com/apache/airflow/commit/1dcd3e13fd0a078fc9440e91b77f6f87aa60dd3b>`_ 2020-12-05 ``Add support for extra links coming from the providers (#12472)``
-`c34ef853c8 <https://github.com/apache/airflow/commit/c34ef853c890e08f5468183c03dc8f3f3ce84af2>`_ 2020-11-20 ``Separate out documentation building per provider (#12444)``
-`0080354502 <https://github.com/apache/airflow/commit/00803545023b096b8db4fbd6eb473843096d7ce4>`_ 2020-11-18 ``Update provider READMEs for 1.0.0b2 batch release (#12449)``
-`ae7cb4a1e2 <https://github.com/apache/airflow/commit/ae7cb4a1e2a96351f1976cf5832615e24863e05d>`_ 2020-11-17 ``Update wrong commit hash in backport provider changes (#12390)``
-`6889a333cf <https://github.com/apache/airflow/commit/6889a333cff001727eb0a66e375544a28c9a5f03>`_ 2020-11-15 ``Improvements for operators and hooks ref docs (#12366)``
-`7825e8f590 <https://github.com/apache/airflow/commit/7825e8f59034645ab3247229be83a3aa90baece1>`_ 2020-11-13 ``Docs installation improvements (#12304)``
-`85a18e13d9 <https://github.com/apache/airflow/commit/85a18e13d9dec84275283ff69e34704b60d54a75>`_ 2020-11-09 ``Point at pypi project pages for cross-dependency of provider packages (#12212)``
-`59eb5de78c <https://github.com/apache/airflow/commit/59eb5de78c70ee9c7ae6e4cba5c7a2babb8103ca>`_ 2020-11-09 ``Update provider READMEs for up-coming 1.0.0beta1 releases (#12206)``
-`b2a28d1590 <https://github.com/apache/airflow/commit/b2a28d1590410630d66966aa1f2b2a049a8c3b32>`_ 2020-11-09 ``Moves provider packages scripts to dev (#12082)``
-`41bf172c1d <https://github.com/apache/airflow/commit/41bf172c1dc75099f4f9d8b3f3350b4b1f523ef9>`_ 2020-11-04 ``Simplify string expressions (#12093)``
-`4e8f9cc8d0 <https://github.com/apache/airflow/commit/4e8f9cc8d02b29c325b8a5a76b4837671bdf5f68>`_ 2020-11-03 ``Enable Black - Python Auto Formmatter (#9550)``
-`8c42cf1b00 <https://github.com/apache/airflow/commit/8c42cf1b00c90f0d7f11b8a3a455381de8e003c5>`_ 2020-11-03 ``Use PyUpgrade to use Python 3.6 features (#11447)``
-`5a439e84eb <https://github.com/apache/airflow/commit/5a439e84eb6c0544dc6c3d6a9f4ceeb2172cd5d0>`_ 2020-10-26 ``Prepare providers release 0.0.2a1 (#11855)``
-`872b1566a1 <https://github.com/apache/airflow/commit/872b1566a11cb73297e657ff325161721b296574>`_ 2020-10-25 ``Generated backport providers readmes/setup for 2020.10.29 (#11826)``
-`349b0811c3 <https://github.com/apache/airflow/commit/349b0811c3022605426ba57d30936240a7c2848a>`_ 2020-10-20 ``Add D200 pydocstyle check (#11688)``
-`16e7129719 <https://github.com/apache/airflow/commit/16e7129719f1c0940aef2a93bed81368e997a746>`_ 2020-10-13 ``Added support for provider packages for Airflow 2.0 (#11487)``
-`0a0e1af800 <https://github.com/apache/airflow/commit/0a0e1af80038ef89974c3c8444461fe867945daa>`_ 2020-10-03 ``Fix Broken Markdown links in Providers README TOC (#11249)``
-`ca4238eb4d <https://github.com/apache/airflow/commit/ca4238eb4d9a2aef70eb641343f59ee706d27d13>`_ 2020-10-02 ``Fixed month in backport packages to October (#11242)``
-`5220e4c384 <https://github.com/apache/airflow/commit/5220e4c3848a2d2c81c266ef939709df9ce581c5>`_ 2020-10-02 ``Prepare Backport release 2020.09.07 (#11238)``
-`720912f67b <https://github.com/apache/airflow/commit/720912f67b3af0bdcbac64d6b8bf6d51c6247e26>`_ 2020-10-02 ``Strict type check for multiple providers (#11229)``
-`c58d60635d <https://github.com/apache/airflow/commit/c58d60635dbab1a91f38e989f72f91645cb7eb62>`_ 2020-09-11 ``Update qubole_hook to not remove pool as an arg for qubole_operator (#10820)``
-`9549274d11 <https://github.com/apache/airflow/commit/9549274d110f689a0bd709db829a4d69e274eed9>`_ 2020-09-09 ``Upgrade black to 20.8b1 (#10818)``
-`36aa88ffc1 <https://github.com/apache/airflow/commit/36aa88ffc1e3feb5c6f4520871a4f6e3196c0804>`_ 2020-09-03 ``Add jupytercmd and fix task failure when notify set as true in qubole operator (#10599)``
-`fdd9b6f65b <https://github.com/apache/airflow/commit/fdd9b6f65b608c516b8a062b058972d9a45ec9e3>`_ 2020-08-25 ``Enable Black on Providers Packages (#10543)``
-`d1bce91bb2 <https://github.com/apache/airflow/commit/d1bce91bb21d5a468fa6a0207156c28fe1ca6513>`_ 2020-08-25 ``PyDocStyle: Enable D403: Capitalized first word of docstring (#10530)``
-`3696c34c28 <https://github.com/apache/airflow/commit/3696c34c28c6bc7b442deab999d9ecba24ed0e34>`_ 2020-08-24 ``Fix typo in the word "release" (#10528)``
-`ee7ca128a1 <https://github.com/apache/airflow/commit/ee7ca128a17937313566f2badb6cc569c614db94>`_ 2020-08-22 ``Fix broken Markdown refernces in Providers README (#10483)``
-`cdec301254 <https://github.com/apache/airflow/commit/cdec3012542b45d23a05f62d69110944ba542e2a>`_ 2020-08-07 ``Add correct signature to all operators and sensors (#10205)``
-`24c8e4c2d6 <https://github.com/apache/airflow/commit/24c8e4c2d6e359ecc2c7d6275dccc68de4a82832>`_ 2020-08-06 ``Changes to all the constructors to remove the args argument (#10163)``
-`aeea71274d <https://github.com/apache/airflow/commit/aeea71274d4527ff2351102e94aa38bda6099e7f>`_ 2020-08-02 ``Remove 'args' parameter from provider operator constructors (#10097)``
-`7d24b088cd <https://github.com/apache/airflow/commit/7d24b088cd736cfa18f9214e4c9d6ce2d5865f3d>`_ 2020-07-25 ``Stop using start_date in default_args in example_dags (2) (#9985)``
-`33f0cd2657 <https://github.com/apache/airflow/commit/33f0cd2657b2e77ea3477e0c93f13f1474be628e>`_ 2020-07-22 ``apply_default keeps the function signature for mypy (#9784)``
-`3190db5246 <https://github.com/apache/airflow/commit/3190db52469f9d9a338231a9e8e7f333a6fbb638>`_ 2020-06-24 ``[AIRFLOW-9347] Fix QuboleHook unable to add list to tags (#9349)``
-`d0e7db4024 <https://github.com/apache/airflow/commit/d0e7db4024806af35e3c9a2cae460fdeedd4d2ec>`_ 2020-06-19 ``Fixed release number for fresh release (#9408)``
-`12af6a0800 <https://github.com/apache/airflow/commit/12af6a08009b8776e00d8a0aab92363eb8c4e8b1>`_ 2020-06-19 ``Final cleanup for 2020.6.23rc1 release preparation (#9404)``
-`c7e5bce57f <https://github.com/apache/airflow/commit/c7e5bce57fe7f51cefce4f8a41ce408ac5675d13>`_ 2020-06-19 ``Prepare backport release candidate for 2020.6.23rc1 (#9370)``
-`f6bd817a3a <https://github.com/apache/airflow/commit/f6bd817a3aac0a16430fc2e3d59c1f17a69a15ac>`_ 2020-06-16 ``Introduce 'transfers' packages (#9320)``
-`0b0e4f7a4c <https://github.com/apache/airflow/commit/0b0e4f7a4cceff3efe15161fb40b984782760a34>`_ 2020-05-26 ``Preparing for RC3 relase of backports (#9026)``
-`00642a46d0 <https://github.com/apache/airflow/commit/00642a46d019870c4decb3d0e47c01d6a25cb88c>`_ 2020-05-26 ``Fixed name of 20 remaining wrongly named operators. (#8994)``
-`1d36b0303b <https://github.com/apache/airflow/commit/1d36b0303b8632fce6de78ca4e782ae26ee06fea>`_ 2020-05-23 ``Fix references in docs (#8984)``
-`375d1ca229 <https://github.com/apache/airflow/commit/375d1ca229464617780623c61c6e8a1bf570c87f>`_ 2020-05-19 ``Release candidate 2 for backport packages 2020.05.20 (#8898)``
-`12c5e5d8ae <https://github.com/apache/airflow/commit/12c5e5d8ae25fa633efe63ccf4db389e2b796d79>`_ 2020-05-17 ``Prepare release candidate for backport packages (#8891)``
-`f3521fb0e3 <https://github.com/apache/airflow/commit/f3521fb0e36733d8bd356123e56a453fd37a6dca>`_ 2020-05-16 ``Regenerate readme files for backport package release (#8886)``
-`92585ca4cb <https://github.com/apache/airflow/commit/92585ca4cb375ac879f4ab331b3a063106eb7b92>`_ 2020-05-15 ``Added automated release notes generation for backport operators (#8807)``
-`4b06fde0f1 <https://github.com/apache/airflow/commit/4b06fde0f10ce178b3c336c5d901e3b089f2863d>`_ 2020-05-12 ``Fix Flake8 errors (#8841)``
-`87969a350d <https://github.com/apache/airflow/commit/87969a350ddd41e9e77776af6d780b31e363eaca>`_ 2020-04-09 ``[AIRFLOW-6515] Change Log Levels from Info/Warn to Error (#8170)``
-`4bde99f132 <https://github.com/apache/airflow/commit/4bde99f1323d72f6c84c1548079d5e98fc0a2a9a>`_ 2020-03-23 ``Make airflow/providers pylint compatible (#7802)``
-`05443c6dc8 <https://github.com/apache/airflow/commit/05443c6dc8100e791446bbcc0df04de6e34017bb>`_ 2020-03-23 ``Add missing call to Super class in remaining providers (#7828)``
-`de7e934ca3 <https://github.com/apache/airflow/commit/de7e934ca3f21ce82f67accf92811b3ac044476f>`_ 2020-03-17 ``[AIRFLOW-7079] Remove redundant code for storing template_fields (#7750)``
-`3320e432a1 <https://github.com/apache/airflow/commit/3320e432a129476dbc1c55be3b3faa3326a635bc>`_ 2020-02-24 ``[AIRFLOW-6817] Lazy-load 'airflow.DAG' to keep user-facing API untouched (#7517)``
-`4d03e33c11 <https://github.com/apache/airflow/commit/4d03e33c115018e30fa413c42b16212481ad25cc>`_ 2020-02-22 ``[AIRFLOW-6817] remove imports from 'airflow/__init__.py', replaced implicit imports with explicit imports, added entry to 'UPDATING.MD' - squashed/rebased (#7456)``
-`9cbd7de6d1 <https://github.com/apache/airflow/commit/9cbd7de6d115795aba8bfb8addb060bfdfbdf87b>`_ 2020-02-18 ``[AIRFLOW-6792] Remove _operator/_hook/_sensor in providers package and add tests (#7412)``
-`f3ad5cf618 <https://github.com/apache/airflow/commit/f3ad5cf6185b9d406d0fb0a4ecc0b5536f79217a>`_ 2020-02-03 ``[AIRFLOW-4681] Make sensors module pylint compatible (#7309)``
-`97a429f9d0 <https://github.com/apache/airflow/commit/97a429f9d0cf740c5698060ad55f11e93cb57b55>`_ 2020-02-02 ``[AIRFLOW-6714] Remove magic comments about UTF-8 (#7338)``
-`cf141506a2 <https://github.com/apache/airflow/commit/cf141506a25dbba279b85500d781f7e056540721>`_ 2020-02-02 ``[AIRFLOW-6708] Set unique logger names (#7330)``
-`a2d6a2f85e <https://github.com/apache/airflow/commit/a2d6a2f85e07c38be479e91e4a27981f308f4711>`_ 2020-01-31 ``[AIRFLOW-6687] Switch kubernetes tests to example_dags (#7299)``
-`83c037873f <https://github.com/apache/airflow/commit/83c037873ff694eed67ba8b30f2d9c88b2c7c6f2>`_ 2020-01-30 ``[AIRFLOW-6674] Move example_dags in accordance with AIP-21 (#7287)``
-`c42a375e79 <https://github.com/apache/airflow/commit/c42a375e799e5adb3f9536616372dc90ff47e6c8>`_ 2020-01-27 ``[AIRFLOW-6644][AIP-21] Move service classes to providers package (#7265)``
-================================================================================================= =========== ======================================================================================================================================================================
diff --git a/docs/apache-airflow-providers-qubole/index.rst b/docs/apache-airflow-providers-qubole/index.rst
deleted file mode 100644
index 1d51169c4f..0000000000
--- a/docs/apache-airflow-providers-qubole/index.rst
+++ /dev/null
@@ -1,112 +0,0 @@
-
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-``apache-airflow-providers-qubole``
-===================================
-
-
-.. toctree::
- :hidden:
- :maxdepth: 1
- :caption: Basics
-
- Home <self>
- Changelog <changelog>
- Security <security>
-
-.. toctree::
- :hidden:
- :maxdepth: 1
- :caption: Guides
-
- Operators <operators/index>
-
-.. toctree::
- :hidden:
- :maxdepth: 1
- :caption: References
-
- Python API <_api/airflow/providers/qubole/index>
-
-.. toctree::
- :hidden:
- :maxdepth: 1
- :caption: System tests
-
- System Tests <_api/tests/system/providers/qubole/index>
-
-.. toctree::
- :hidden:
- :maxdepth: 1
- :caption: Resources
-
- Example DAGs <https://github.com/apache/airflow/tree/providers-qubole/|version|/tests/system/providers/qubole>
- PyPI Repository <https://pypi.org/project/apache-airflow-providers-qubole/>
- Installing from sources <installing-providers-from-sources>
-
-
-.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME!
-.. toctree::
- :hidden:
- :maxdepth: 1
- :caption: Commits
-
- Detailed list of commits <commits>
-
-
-Package apache-airflow-providers-qubole
-------------------------------------------------------
-
-`Qubole <https://www.qubole.com/>`__
-
-
-Release: 3.4.3
-
-Provider package
-----------------
-
-This is a provider package for ``qubole`` provider. All classes for this provider package
-are in ``airflow.providers.qubole`` python package.
-
- .. warning::
-
- This provider is not maintained anymore by the community. It has been removed and is not going to be
- updated anymore. The removal was done according to the process described in
- `Removing community providers <https://github.com/apache/airflow/blob/main/PROVIDERS.rst#removing-community-providers>`_
-
- Feel free to contact Airflow Development Mailing List if you have any questions.
-
-Installation
-------------
-
-You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below)
-for the minimum Airflow version supported) via
-``pip install apache-airflow-providers-qubole``
-
-Requirements
-------------
-
-The minimum Apache Airflow version supported by this provider package is ``2.5.0``.
-
-======================================= ==================
-PIP package Version required
-======================================= ==================
-``apache-airflow`` ``>=2.5.0``
-``apache-airflow-providers-common-sql`` ``>=1.3.1``
-``qds-sdk`` ``>=1.10.4``
-======================================= ==================
diff --git a/docs/apache-airflow-providers-qubole/installing-providers-from-sources.rst b/docs/apache-airflow-providers-qubole/installing-providers-from-sources.rst
deleted file mode 100644
index b4e730f4ff..0000000000
--- a/docs/apache-airflow-providers-qubole/installing-providers-from-sources.rst
+++ /dev/null
@@ -1,18 +0,0 @@
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-.. include:: ../exts/includes/installing-providers-from-sources.rst
diff --git a/docs/apache-airflow-providers-qubole/operators/index.rst b/docs/apache-airflow-providers-qubole/operators/index.rst
deleted file mode 100644
index 1dbccd0544..0000000000
--- a/docs/apache-airflow-providers-qubole/operators/index.rst
+++ /dev/null
@@ -1,32 +0,0 @@
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-
-
-Qubole Operators
-================
-
-
-.. toctree::
- :maxdepth: 1
- :glob:
-
- *
-
-.. note::
- You can learn how to use Google Cloud integrations by analyzing the
- `source code <https://github.com/apache/airflow/tree/providers-qubole/|version|/tests/system/providers/qubole/>`_ of the particular example DAGs.
diff --git a/docs/apache-airflow-providers-qubole/operators/qubole.rst b/docs/apache-airflow-providers-qubole/operators/qubole.rst
deleted file mode 100644
index b95e553ed2..0000000000
--- a/docs/apache-airflow-providers-qubole/operators/qubole.rst
+++ /dev/null
@@ -1,184 +0,0 @@
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-
-Qubole
-======
-`Qubole <https://www.qubole.com/>`__ is an open, simple, and secure data lake platform for machine learning, streaming and adhoc analytics.
-Qubole delivers a Self-Service Platform for Big Data Analytics built on Amazon Web Services, Microsoft and Google Clouds.
-
-Airflow provides operators to execute tasks (commands) on QDS and perform checks against Qubole Commands.
-Also, there are provided sensors that waits for a file, folder or partition to be present in cloud storage and check for its presence via QDS APIs
-
-
-.. _howto/operator:QuboleOperator:
-
-Execute tasks
-^^^^^^^^^^^^^
-
-To run following commands use
-:class:`~airflow.providers.qubole.operators.qubole.QuboleOperator`.
-
-Run Hive command
-""""""""""""""""
-
-To run query that shows all tables you can use
-
-.. exampleinclude:: /../../tests/system/providers/qubole/example_qubole.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_qubole_run_hive_query]
- :end-before: [END howto_operator_qubole_run_hive_query]
-
-Also you can run script that locates in the bucket by passing path to query file
-
-.. exampleinclude:: /../../tests/system/providers/qubole/example_qubole.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_qubole_run_hive_script]
- :end-before: [END howto_operator_qubole_run_hive_script]
-
-Run Hadoop command
-""""""""""""""""""
-
-To run jar file in your Hadoop cluster use
-
-.. exampleinclude:: /../../tests/system/providers/qubole/example_qubole.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_qubole_run_hadoop_jar]
- :end-before: [END howto_operator_qubole_run_hadoop_jar]
-
-Run Pig command
-"""""""""""""""
-
-To run script in *Pig Latin* in your Hadoop cluster use
-
-.. exampleinclude:: /../../tests/system/providers/qubole/example_qubole.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_qubole_run_pig_script]
- :end-before: [END howto_operator_qubole_run_pig_script]
-
-Run Shell command
-"""""""""""""""""
-
-To run Shell-script use
-
-.. exampleinclude:: /../../tests/system/providers/qubole/example_qubole.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_qubole_run_shell_script]
- :end-before: [END howto_operator_qubole_run_shell_script]
-
-Run Presto command
-""""""""""""""""""
-
-To run query using Presto use
-
-.. exampleinclude:: /../../tests/system/providers/qubole/example_qubole.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_qubole_run_presto_query]
- :end-before: [END howto_operator_qubole_run_presto_query]
-
-Run DB commands
-"""""""""""""""
-
-To run query as `DbTap <https://docs.qubole.com/en/latest/rest-api/dbtap_api/index.html>`_ use
-
-.. exampleinclude:: /../../tests/system/providers/qubole/example_qubole.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_qubole_run_db_tap_query]
- :end-before: [END howto_operator_qubole_run_db_tap_query]
-
-To run DB export command use
-
-.. exampleinclude:: /../../tests/system/providers/qubole/example_qubole.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_qubole_run_db_export]
- :end-before: [END howto_operator_qubole_run_db_export]
-
-To run DB import command use
-
-.. exampleinclude:: /../../tests/system/providers/qubole/example_qubole.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_qubole_run_db_import]
- :end-before: [END howto_operator_qubole_run_db_import]
-
-Run Spark commands
-""""""""""""""""""
-
-To run Scala script as a Spark job use
-
-.. exampleinclude:: /../../tests/system/providers/qubole/example_qubole.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_qubole_run_spark_scala]
- :end-before: [END howto_operator_qubole_run_spark_scala]
-
-
-.. _howto/operator:QuboleFileSensor:
-
-File sensor
-^^^^^^^^^^^
-
-Usage examples of
-:class:`~airflow.providers.qubole.sensors.qubole.QuboleFileSensor`.
-
-File or directory existence
-"""""""""""""""""""""""""""
-
-To wait for file or directory existence in cluster use
-
-.. exampleinclude:: /../../tests/system/providers/qubole/example_qubole_sensors.py
- :language: python
- :dedent: 4
- :start-after: [START howto_sensor_qubole_run_file_sensor]
- :end-before: [END howto_sensor_qubole_run_file_sensor]
-
-
-.. _howto/operator:QubolePartitionSensor:
-
-Partition sensor
-^^^^^^^^^^^^^^^^
-
-Usage examples of
-:class:`~airflow.providers.qubole.sensors.qubole.QubolePartitionSensor`.
-
-Partition existence
-"""""""""""""""""""
-
-To wait for table partition existence in cluster use
-
-.. exampleinclude:: /../../tests/system/providers/qubole/example_qubole_sensors.py
- :language: python
- :dedent: 4
- :start-after: [START howto_sensor_qubole_run_partition_sensor]
- :end-before: [END howto_sensor_qubole_run_partition_sensor]
-
-
-Reference
-^^^^^^^^^
-
-For further information, look at:
-
-* `Qubole Data Service Python SDK <https://github.com/qubole/qds-sdk-py>`__
-* `Product Documentation <https://docs.qubole.com/en/latest/>`__
diff --git a/docs/apache-airflow-providers-qubole/operators/qubole_check.rst b/docs/apache-airflow-providers-qubole/operators/qubole_check.rst
deleted file mode 100644
index 482b85d375..0000000000
--- a/docs/apache-airflow-providers-qubole/operators/qubole_check.rst
+++ /dev/null
@@ -1,45 +0,0 @@
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-
-Qubole Check
-============
-`Qubole <https://www.qubole.com/>`__ is an open, simple, and secure data lake platform for machine learning, streaming and adhoc analytics.
-Qubole delivers a Self-Service Platform for Big Data Analytics built on Amazon Web Services, Microsoft and Google Clouds.
-
-Airflow provides operators to execute tasks (commands) on QDS and perform checks against Qubole Commands.
-
-.. _howto/operator:QuboleCheckOperator:
-
-Perform checks
-^^^^^^^^^^^^^^
-
-To performs checks against Qubole Commands you can use
-:class:`~airflow.providers.qubole.operators.qubole.QuboleCheckOperator`.
-
-Operator expects a command that will be executed on QDS.
-By default, each value on first row of the result of this Qubole Command is evaluated using python ``bool`` casting.
-If any of the values return ``False``, the check is failed and errors out.
-
-
-Reference
-^^^^^^^^^
-
-For further information, look at:
-
-* `Qubole Data Service Python SDK <https://github.com/qubole/qds-sdk-py>`__
-* `Product Documentation <https://docs.qubole.com/en/latest/>`__
diff --git a/docs/apache-airflow-providers-qubole/security.rst b/docs/apache-airflow-providers-qubole/security.rst
deleted file mode 100644
index b8f95e6ecf..0000000000
--- a/docs/apache-airflow-providers-qubole/security.rst
+++ /dev/null
@@ -1,19 +0,0 @@
-
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-.. include:: ../exts/includes/security.rst
diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst
index 050781dc92..3060d7c2db 100644
--- a/docs/apache-airflow/extra-packages-ref.rst
+++ b/docs/apache-airflow/extra-packages-ref.rst
@@ -408,8 +408,6 @@ so there is no replacement for ``crypto`` extra.
+---------------------+-----------------------------+
| pinot | apache.pinot |
+---------------------+-----------------------------+
-| qds | qubole |
-+---------------------+-----------------------------+
| s3 | amazon |
+---------------------+-----------------------------+
| spark | apache.spark |
diff --git a/docs/integration-logos/qubole/Qubole.png b/docs/integration-logos/qubole/Qubole.png
deleted file mode 100644
index 59bfcd2ce1..0000000000
Binary files a/docs/integration-logos/qubole/Qubole.png and /dev/null differ
diff --git a/setup.py b/setup.py
index fcc5428824..ad30baacd1 100644
--- a/setup.py
+++ b/setup.py
@@ -646,7 +646,6 @@ EXTRAS_DEPRECATED_ALIASES: dict[str, str] = {
"kubernetes": "cncf.kubernetes",
"mssql": "microsoft.mssql",
"pinot": "apache.pinot",
- "qds": "qubole",
"s3": "amazon",
"spark": "apache.spark",
"webhdfs": "apache.webhdfs",
diff --git a/tests/providers/qubole/__init__.py b/tests/providers/qubole/__init__.py
deleted file mode 100644
index 217e5db960..0000000000
--- a/tests/providers/qubole/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/tests/providers/qubole/hooks/__init__.py b/tests/providers/qubole/hooks/__init__.py
deleted file mode 100644
index 217e5db960..0000000000
--- a/tests/providers/qubole/hooks/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/tests/providers/qubole/hooks/test_qubole.py b/tests/providers/qubole/hooks/test_qubole.py
deleted file mode 100644
index a037b8110a..0000000000
--- a/tests/providers/qubole/hooks/test_qubole.py
+++ /dev/null
@@ -1,80 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-from unittest import mock
-
-from qds_sdk.commands import PrestoCommand
-
-from airflow.providers.qubole.hooks.qubole import QuboleHook
-
-DAG_ID = "qubole_test_dag"
-TASK_ID = "test_task"
-RESULTS_WITH_HEADER = "header1\theader2\nval1\tval2"
-RESULTS_WITH_NO_HEADER = "val1\tval2"
-
-add_tags = QuboleHook._add_tags
-
-
-def get_result_mock(fp, inline, delim, fetch, arguments):
- if arguments[0] == "true":
- fp.write(bytearray(RESULTS_WITH_HEADER, "utf-8"))
- else:
- fp.write(bytearray(RESULTS_WITH_NO_HEADER, "utf-8"))
-
-
-class TestQuboleHook:
- def test_add_string_to_tags(self):
- tags = {"dag_id", "task_id"}
- add_tags(tags, "string")
- assert {"dag_id", "task_id", "string"} == tags
-
- def test_add_list_to_tags(self):
- tags = {"dag_id", "task_id"}
- add_tags(tags, ["value1", "value2"])
- assert {"dag_id", "task_id", "value1", "value2"} == tags
-
- def test_add_tuple_to_tags(self):
- tags = {"dag_id", "task_id"}
- add_tags(tags, ("value1", "value2"))
- assert {"dag_id", "task_id", "value1", "value2"} == tags
-
- @mock.patch("qds_sdk.commands.Command.get_results", new=get_result_mock)
- def test_get_results_with_headers(self):
- dag = mock.MagicMock()
- dag.dag_id = DAG_ID
- hook = QuboleHook(task_id=TASK_ID, command_type="prestocmd", dag=dag)
-
- task = mock.MagicMock()
- task.xcom_pull.return_value = "test_command_id"
- with mock.patch("qds_sdk.resource.Resource.find", return_value=PrestoCommand):
- results = open(hook.get_results(ti=task, include_headers=True)).read()
- assert results == RESULTS_WITH_HEADER
-
- @mock.patch("qds_sdk.commands.Command.get_results", new=get_result_mock)
- def test_get_results_without_headers(self):
- dag = mock.MagicMock()
- dag.dag_id = DAG_ID
- hook = QuboleHook(task_id=TASK_ID, command_type="prestocmd", dag=dag)
-
- task = mock.MagicMock()
- task.xcom_pull.return_value = "test_command_id"
-
- with mock.patch("qds_sdk.resource.Resource.find", return_value=PrestoCommand):
- results = open(hook.get_results(ti=task, include_headers=False)).read()
- assert results == RESULTS_WITH_NO_HEADER
diff --git a/tests/providers/qubole/hooks/test_qubole_check.py b/tests/providers/qubole/hooks/test_qubole_check.py
deleted file mode 100644
index 8c5b27da52..0000000000
--- a/tests/providers/qubole/hooks/test_qubole_check.py
+++ /dev/null
@@ -1,42 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-from airflow.providers.qubole.hooks.qubole_check import parse_first_row
-
-
-class TestQuboleCheckHook:
- def test_single_row_bool(self):
- query_result = ["true\ttrue"]
- record_list = parse_first_row(query_result)
- assert [True, True] == record_list
-
- def test_multi_row_bool(self):
- query_result = ["true\tfalse", "true\tfalse"]
- record_list = parse_first_row(query_result)
- assert [True, False] == record_list
-
- def test_single_row_float(self):
- query_result = ["0.23\t34"]
- record_list = parse_first_row(query_result)
- assert [0.23, 34] == record_list
-
- def test_single_row_mixed_types(self):
- query_result = ["name\t44\t0.23\tTrue"]
- record_list = parse_first_row(query_result)
- assert ["name", 44, 0.23, True] == record_list
diff --git a/tests/providers/qubole/operators/__init__.py b/tests/providers/qubole/operators/__init__.py
deleted file mode 100644
index 217e5db960..0000000000
--- a/tests/providers/qubole/operators/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/tests/providers/qubole/operators/test_qubole.py b/tests/providers/qubole/operators/test_qubole.py
deleted file mode 100644
index a3cb581133..0000000000
--- a/tests/providers/qubole/operators/test_qubole.py
+++ /dev/null
@@ -1,179 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-from unittest import mock
-
-import pytest
-
-from airflow import settings
-from airflow.models import DAG, Connection
-from airflow.providers.qubole.hooks.qubole import QuboleHook
-from airflow.providers.qubole.operators.qubole import QDSLink, QuboleOperator
-from airflow.serialization.serialized_objects import SerializedDAG
-from airflow.utils import db
-from airflow.utils.timezone import datetime
-
-DAG_ID = "qubole_test_dag"
-TASK_ID = "test_task"
-DEFAULT_CONN = "qubole_default"
-TEMPLATE_CONN = "my_conn_id"
-TEST_CONN = "qubole_test_conn"
-DEFAULT_DATE = datetime(2017, 1, 1)
-
-
-class TestQuboleOperator:
- def setup_method(self):
- db.merge_conn(Connection(conn_id=DEFAULT_CONN, conn_type="HTTP"))
- db.merge_conn(Connection(conn_id=TEST_CONN, conn_type="HTTP", host="http://localhost/api"))
-
- def teardown_method(self):
- session = settings.Session()
- session.query(Connection).filter(Connection.conn_id == TEST_CONN).delete()
- session.flush()
- session.close()
-
- def test_init_with_default_connection(self):
- op = QuboleOperator(task_id=TASK_ID)
- assert op.task_id == TASK_ID
- assert op.qubole_conn_id == DEFAULT_CONN
-
- def test_init_with_template_connection(self):
- with DAG(DAG_ID, start_date=DEFAULT_DATE):
- task = QuboleOperator(task_id=TASK_ID, qubole_conn_id="{{ qubole_conn_id }}")
-
- task.render_template_fields({"qubole_conn_id": TEMPLATE_CONN})
- assert task.task_id == TASK_ID
- assert task.qubole_conn_id == TEMPLATE_CONN
-
- def test_init_with_template_cluster_label(self, create_task_instance_of_operator):
- ti = create_task_instance_of_operator(
- QuboleOperator,
- dag_id="test_init_with_template_cluster_label",
- execution_date=DEFAULT_DATE,
- task_id=TASK_ID,
- cluster_label="{{ params.cluster_label }}",
- params={"cluster_label": "default"},
- )
- ti.render_templates()
- assert ti.task.cluster_label == "default"
-
- def test_get_hook(self):
- dag = DAG(DAG_ID, start_date=DEFAULT_DATE)
-
- with dag:
- task = QuboleOperator(task_id=TASK_ID, command_type="hivecmd", dag=dag)
-
- hook = task.get_hook()
- assert hook.__class__ == QuboleHook
-
- def test_hyphen_args_note_id(self):
- dag = DAG(DAG_ID, start_date=DEFAULT_DATE)
-
- with dag:
- task = QuboleOperator(task_id=TASK_ID, command_type="sparkcmd", note_id="123", dag=dag)
-
- assert task.get_hook().create_cmd_args({"run_id": "dummy"})[0] == "--note-id=123"
-
- def test_notify(self):
- dag = DAG(DAG_ID, start_date=DEFAULT_DATE)
-
- with dag:
- task = QuboleOperator(task_id=TASK_ID, command_type="sparkcmd", notify=True, dag=dag)
-
- assert task.get_hook().create_cmd_args({"run_id": "dummy"})[0] == "--notify"
-
- def test_position_args_parameters(self):
- dag = DAG(DAG_ID, start_date=DEFAULT_DATE)
-
- with dag:
- task = QuboleOperator(
- task_id=TASK_ID, command_type="pigcmd", parameters="key1=value1 key2=value2", dag=dag
- )
-
- assert task.get_hook().create_cmd_args({"run_id": "dummy"})[1] == "key1=value1"
- assert task.get_hook().create_cmd_args({"run_id": "dummy"})[2] == "key2=value2"
-
- cmd = "s3distcp --src s3n://airflow/source_hadoopcmd --dest s3n://airflow/destination_hadoopcmd"
- task = QuboleOperator(task_id=TASK_ID + "_1", command_type="hadoopcmd", dag=dag, sub_command=cmd)
-
- assert task.get_hook().create_cmd_args({"run_id": "dummy"})[1] == "s3distcp"
- assert task.get_hook().create_cmd_args({"run_id": "dummy"})[2] == "--src"
- assert task.get_hook().create_cmd_args({"run_id": "dummy"})[3] == "s3n://airflow/source_hadoopcmd"
- assert task.get_hook().create_cmd_args({"run_id": "dummy"})[4] == "--dest"
- assert (
- task.get_hook().create_cmd_args({"run_id": "dummy"})[5] == "s3n://airflow/destination_hadoopcmd"
- )
-
- def test_get_link(self, create_task_instance_of_operator):
- ti = create_task_instance_of_operator(
- QuboleOperator,
- dag_id="test_get_link",
- execution_date=DEFAULT_DATE,
- task_id=TASK_ID,
- qubole_conn_id=TEST_CONN,
- command_type="shellcmd",
- parameters="param1 param2",
- )
- ti.xcom_push("qbol_cmd_id", 12345)
-
- url = ti.task.get_extra_links(ti, "Go to QDS")
- assert url == "http://localhost/v2/analyze?command_id=12345"
-
- @pytest.mark.need_serialized_dag
- def test_extra_serialized_field(self, dag_maker, create_task_instance_of_operator):
- ti = create_task_instance_of_operator(
- QuboleOperator,
- dag_id="test_extra_serialized_field",
- execution_date=DEFAULT_DATE,
- task_id=TASK_ID,
- command_type="shellcmd",
- qubole_conn_id=TEST_CONN,
- )
-
- serialized_dag = dag_maker.get_serialized_data()
- assert "qubole_conn_id" in serialized_dag["dag"]["tasks"][0]
-
- dag = SerializedDAG.from_dict(serialized_dag)
- simple_task = dag.task_dict[TASK_ID]
- assert getattr(simple_task, "qubole_conn_id") == TEST_CONN
-
- assert isinstance(list(simple_task.operator_extra_links)[0], QDSLink)
-
- ti.xcom_push("qbol_cmd_id", 12345)
- url = simple_task.get_extra_links(ti, "Go to QDS")
- assert url == "http://localhost/v2/analyze?command_id=12345"
-
- def test_parameter_pool_passed(self):
- test_pool = "test_pool"
- op = QuboleOperator(task_id=TASK_ID, pool=test_pool)
- assert op.pool == test_pool
-
- @mock.patch("airflow.providers.qubole.hooks.qubole.QuboleHook.get_results")
- def test_parameter_include_header_passed(self, mock_get_results):
- dag = DAG(DAG_ID, start_date=DEFAULT_DATE)
- qubole_operator = QuboleOperator(task_id=TASK_ID, dag=dag, command_type="prestocmd")
- qubole_operator.get_results(include_headers=True)
- mock_get_results.asset_called_with("include_headers", True)
-
- @mock.patch("airflow.providers.qubole.hooks.qubole.QuboleHook.get_results")
- def test_parameter_include_header_missing(self, mock_get_results):
- dag = DAG(DAG_ID, start_date=DEFAULT_DATE)
- qubole_operator = QuboleOperator(task_id=TASK_ID, dag=dag, command_type="prestocmd")
- qubole_operator.get_results()
- mock_get_results.asset_called_with("include_headers", False)
diff --git a/tests/providers/qubole/operators/test_qubole_check.py b/tests/providers/qubole/operators/test_qubole_check.py
deleted file mode 100644
index b37cf732d9..0000000000
--- a/tests/providers/qubole/operators/test_qubole_check.py
+++ /dev/null
@@ -1,192 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-from datetime import datetime
-from unittest import mock
-from unittest.mock import MagicMock
-
-import pytest
-from qds_sdk.commands import HiveCommand
-
-from airflow.exceptions import AirflowException
-from airflow.models import DAG
-from airflow.providers.common.sql.operators.sql import SQLCheckOperator, SQLValueCheckOperator
-from airflow.providers.qubole.hooks.qubole import QuboleHook
-from airflow.providers.qubole.hooks.qubole_check import QuboleCheckHook
-from airflow.providers.qubole.operators.qubole_check import (
- QuboleCheckOperator,
- QuboleValueCheckOperator,
- _QuboleCheckOperatorMixin,
-)
-
-
-@pytest.mark.parametrize(
- "operator_class, kwargs, parent_check_operator",
- [
- (QuboleCheckOperator, dict(sql="Select * from test_table"), SQLCheckOperator),
- (
- QuboleValueCheckOperator,
- dict(sql="Select * from test_table", pass_value=95),
- SQLValueCheckOperator,
- ),
- ],
-)
-class TestQuboleCheckMixin:
- def setup_method(self):
- self.task_id = "test_task"
-
- def __construct_operator(self, operator_class, **kwargs):
- dag = DAG("test_dag", start_date=datetime(2017, 1, 1))
- return operator_class(task_id=self.task_id, dag=dag, command_type="hivecmd", **kwargs)
-
- def test_get_hook_with_context(self, operator_class, kwargs, parent_check_operator):
- operator = self.__construct_operator(operator_class=operator_class, **kwargs)
- assert isinstance(operator.get_hook(), QuboleCheckHook)
-
- context = {"exec_date": "today"}
- operator._hook_context = context
- hook = operator.get_hook()
- assert hook.context == context
-
- @mock.patch.object(_QuboleCheckOperatorMixin, "get_db_hook")
- @mock.patch.object(_QuboleCheckOperatorMixin, "get_hook")
- def test_get_db_hook(
- self, mock_get_hook, mock_get_db_hook, operator_class, kwargs, parent_check_operator
- ):
- operator = self.__construct_operator(operator_class=operator_class, **kwargs)
- operator.get_db_hook()
- mock_get_db_hook.assert_called_once()
-
- operator.get_hook()
- mock_get_hook.assert_called_once()
-
- def test_execute(self, operator_class, kwargs, parent_check_operator):
- operator = self.__construct_operator(operator_class=operator_class, **kwargs)
-
- with mock.patch.object(parent_check_operator, "execute") as mock_execute:
- operator.execute(context=MagicMock())
- mock_execute.assert_called_once()
-
- @mock.patch("airflow.providers.qubole.operators.qubole_check.handle_airflow_exception")
- def test_execute_fail(self, mock_handle_airflow_exception, operator_class, kwargs, parent_check_operator):
- operator = self.__construct_operator(operator_class=operator_class, **kwargs)
-
- with mock.patch.object(parent_check_operator, "execute") as mock_execute:
- mock_execute.side_effect = AirflowException()
- operator.execute(context=MagicMock())
- mock_execute.assert_called_once()
- mock_handle_airflow_exception.assert_called_once()
-
-
-class TestQuboleValueCheckOperator:
- def setup_method(self):
- self.task_id = "test_task"
- self.conn_id = "default_conn"
-
- def __construct_operator(self, query, pass_value, tolerance=None, results_parser_callable=None):
- dag = DAG("test_dag", start_date=datetime(2017, 1, 1))
-
- return QuboleValueCheckOperator(
- dag=dag,
- task_id=self.task_id,
- conn_id=self.conn_id,
- query=query,
- pass_value=pass_value,
- results_parser_callable=results_parser_callable,
- command_type="hivecmd",
- tolerance=tolerance,
- )
-
- def test_pass_value_template(self):
- pass_value_str = "2018-03-22"
- operator = self.__construct_operator("select date from tab1;", "{{ ds }}")
- result = operator.render_template(operator.pass_value, {"ds": pass_value_str})
-
- assert operator.task_id == self.task_id
- assert result == pass_value_str
-
- @mock.patch.object(QuboleValueCheckOperator, "get_hook")
- def test_execute_pass(self, mock_get_hook):
- mock_hook = mock.Mock()
- mock_hook.get_first.return_value = [10]
- mock_get_hook.return_value = mock_hook
-
- query = "select value from tab1 limit 1;"
-
- operator = self.__construct_operator(query, 5, 1)
-
- operator.execute(None)
-
- mock_hook.get_first.assert_called_once_with(query)
-
- @mock.patch.object(QuboleValueCheckOperator, "get_hook")
- def test_execute_assertion_fail(self, mock_get_hook):
- mock_cmd = mock.Mock()
- mock_cmd.status = "done"
- mock_cmd.id = 123
- mock_cmd.is_success = mock.Mock(return_value=HiveCommand.is_success(mock_cmd.status))
-
- mock_hook = mock.Mock()
- mock_hook.get_first.return_value = [11]
- mock_hook.cmd = mock_cmd
- mock_get_hook.return_value = mock_hook
-
- operator = self.__construct_operator("select value from tab1 limit 1;", 5, 1)
-
- with pytest.raises(AirflowException, match="Qubole Command Id: " + str(mock_cmd.id)):
- operator.execute(context=MagicMock())
-
- mock_cmd.is_success.assert_called_once_with(mock_cmd.status)
-
- @mock.patch.object(QuboleValueCheckOperator, "get_hook")
- def test_execute_assert_query_fail(self, mock_get_hook):
- mock_cmd = mock.Mock()
- mock_cmd.status = "error"
- mock_cmd.id = 123
- mock_cmd.is_success = mock.Mock(return_value=HiveCommand.is_success(mock_cmd.status))
-
- mock_hook = mock.Mock()
- mock_hook.get_first.return_value = [11]
- mock_hook.cmd = mock_cmd
- mock_get_hook.return_value = mock_hook
-
- operator = self.__construct_operator("select value from tab1 limit 1;", 5, 1)
-
- with pytest.raises(AirflowException) as ctx:
- operator.execute(context=MagicMock())
-
- assert "Qubole Command Id: " not in str(ctx.value)
- mock_cmd.is_success.assert_called_once_with(mock_cmd.status)
-
- @mock.patch.object(QuboleCheckHook, "get_query_results")
- @mock.patch.object(QuboleHook, "execute")
- def test_results_parser_callable(self, mock_execute, mock_get_query_results):
- mock_execute.return_value = None
-
- pass_value = "pass_value"
- mock_get_query_results.return_value = pass_value
-
- results_parser_callable = mock.Mock()
- results_parser_callable.return_value = [pass_value]
-
- operator = self.__construct_operator(
- "select value from tab1 limit 1;", pass_value, None, results_parser_callable
- )
- operator.execute(context=MagicMock())
- results_parser_callable.assert_called_once_with([pass_value])
diff --git a/tests/providers/qubole/sensors/__init__.py b/tests/providers/qubole/sensors/__init__.py
deleted file mode 100644
index 217e5db960..0000000000
--- a/tests/providers/qubole/sensors/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/tests/providers/qubole/sensors/test_qubole.py b/tests/providers/qubole/sensors/test_qubole.py
deleted file mode 100644
index 02fd29ba7f..0000000000
--- a/tests/providers/qubole/sensors/test_qubole.py
+++ /dev/null
@@ -1,80 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-from datetime import datetime
-from unittest.mock import patch
-
-import pytest
-
-from airflow.exceptions import AirflowException
-from airflow.models import DAG, Connection
-from airflow.providers.qubole.sensors.qubole import QuboleFileSensor, QubolePartitionSensor
-from airflow.utils import db
-
-DAG_ID = "qubole_test_dag"
-TASK_ID = "test_task"
-DEFAULT_CONN = "qubole_default"
-TEMPLATE_CONN = "my_conn_id"
-DEFAULT_DATE = datetime(2017, 1, 1)
-
-
-class TestQuboleSensor:
- def setup_method(self):
- db.merge_conn(Connection(conn_id=DEFAULT_CONN, conn_type="HTTP"))
-
- @patch("airflow.providers.qubole.sensors.qubole.QuboleFileSensor.poke")
- def test_file_sensor(self, patched_poke):
- patched_poke.return_value = True
- sensor = QuboleFileSensor(
- task_id="test_qubole_file_sensor", data={"files": ["s3://some_bucket/some_file"]}
- )
- assert sensor.poke({})
-
- @patch("airflow.providers.qubole.sensors.qubole.QubolePartitionSensor.poke")
- def test_partition_sensor(self, patched_poke):
- patched_poke.return_value = True
-
- sensor = QubolePartitionSensor(
- task_id="test_qubole_partition_sensor",
- data={
- "schema": "default",
- "table": "my_partitioned_table",
- "columns": [{"column": "month", "values": ["1", "2"]}],
- },
- )
-
- assert sensor.poke({})
-
- @patch("airflow.providers.qubole.sensors.qubole.QubolePartitionSensor.poke")
- def test_partition_sensor_error(self, patched_poke):
- patched_poke.return_value = True
-
- dag = DAG(DAG_ID, start_date=DEFAULT_DATE)
-
- with pytest.raises(AirflowException):
- QubolePartitionSensor(
- task_id="test_qubole_partition_sensor",
- poke_interval=1,
- data={
- "schema": "default",
- "table": "my_partitioned_table",
- "columns": [{"column": "month", "values": ["1", "2"]}],
- },
- dag=dag,
- )
diff --git a/tests/system/providers/qubole/__init__.py b/tests/system/providers/qubole/__init__.py
deleted file mode 100644
index 13a83393a9..0000000000
--- a/tests/system/providers/qubole/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/tests/system/providers/qubole/example_qubole.py b/tests/system/providers/qubole/example_qubole.py
deleted file mode 100644
index 42ba0242f8..0000000000
--- a/tests/system/providers/qubole/example_qubole.py
+++ /dev/null
@@ -1,235 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-import filecmp
-import os
-import random
-import textwrap
-from datetime import datetime
-
-from airflow import DAG
-from airflow.decorators import task
-
-try:
- from airflow.operators.empty import EmptyOperator
-except ModuleNotFoundError:
- from airflow.operators.dummy import DummyOperator as EmptyOperator # type: ignore
-
-from airflow.operators.python import BranchPythonOperator
-from airflow.providers.qubole.operators.qubole import QuboleOperator
-from airflow.utils.trigger_rule import TriggerRule
-
-START_DATE = datetime(2021, 1, 1)
-
-ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
-DAG_ID = "example_qubole_operator"
-
-with DAG(
- dag_id=DAG_ID,
- schedule=None,
- start_date=START_DATE,
- tags=["example"],
-) as dag:
- dag.doc_md = textwrap.dedent(
- """
- This is only an example DAG to highlight usage of QuboleOperator in various scenarios,
- some of these tasks may or may not work based on your Qubole account setup.
-
- Run a shell command from Qubole Analyze against your Airflow cluster with following to
- trigger it manually `airflow dags trigger example_qubole_operator`.
-
- *Note: Make sure that connection `qubole_default` is properly set before running this
- example. Also be aware that it might spin up clusters to run these examples.*
- """
- )
-
- @task(trigger_rule=TriggerRule.ALL_DONE)
- def compare_result(hive_show_table, hive_s3_location, ti=None) -> bool:
- """
- Compares the results of two QuboleOperator tasks.
-
- :param hive_show_table: The "hive_show_table" task.
- :param hive_s3_location: The "hive_s3_location" task.
- :param ti: The TaskInstance object.
- :return: True if the files are the same, False otherwise.
- """
- qubole_result_1 = hive_show_table.get_results(ti)
- qubole_result_2 = hive_s3_location.get_results(ti)
- return filecmp.cmp(qubole_result_1, qubole_result_2)
-
- # [START howto_operator_qubole_run_hive_query]
- hive_show_table = QuboleOperator(
- task_id="hive_show_table",
- command_type="hivecmd",
- query="show tables",
- cluster_label="{{ params.cluster_label }}",
- fetch_logs=True,
- # If `fetch_logs`=true, will fetch qubole command logs and concatenate
- # them into corresponding airflow task logs
- tags="airflow_example_run",
- # To attach tags to qubole command, auto attach 3 tags - dag_id, task_id, run_id
- params={
- "cluster_label": "default",
- },
- )
- # [END howto_operator_qubole_run_hive_query]
-
- # [START howto_operator_qubole_run_hive_script]
- hive_s3_location = QuboleOperator(
- task_id="hive_s3_location",
- command_type="hivecmd",
- script_location="s3n://public-qubole/qbol-library/scripts/show_table.hql",
- notify=True,
- tags=["tag1", "tag2"],
- # If the script at s3 location has any qubole specific macros to be replaced
- # macros='[{"date": "{{ ds }}"}, {"name" : "abc"}]',
- )
- # [END howto_operator_qubole_run_hive_script]
-
- options = ["hadoop_jar_cmd", "presto_cmd", "db_query", "spark_cmd"]
-
- branching = BranchPythonOperator(task_id="branching", python_callable=lambda: random.choice(options))
-
- [hive_show_table, hive_s3_location] >> compare_result(hive_s3_location, hive_show_table) >> branching
-
- join = EmptyOperator(task_id="join", trigger_rule=TriggerRule.ONE_SUCCESS)
-
- # [START howto_operator_qubole_run_hadoop_jar]
- hadoop_jar_cmd = QuboleOperator(
- task_id="hadoop_jar_cmd",
- command_type="hadoopcmd",
- sub_command="jar s3://paid-qubole/HadoopAPIExamples/"
- "jars/hadoop-0.20.1-dev-streaming.jar "
- "-mapper wc "
- "-numReduceTasks 0 -input s3://paid-qubole/HadoopAPITests/"
- "data/3.tsv -output "
- "s3://paid-qubole/HadoopAPITests/data/3_wc",
- cluster_label="{{ params.cluster_label }}",
- fetch_logs=True,
- params={
- "cluster_label": "default",
- },
- )
- # [END howto_operator_qubole_run_hadoop_jar]
-
- # [START howto_operator_qubole_run_pig_script]
- pig_cmd = QuboleOperator(
- task_id="pig_cmd",
- command_type="pigcmd",
- script_location="s3://public-qubole/qbol-library/scripts/script1-hadoop-s3-small.pig",
- parameters="key1=value1 key2=value2",
- )
- # [END howto_operator_qubole_run_pig_script]
-
- branching >> hadoop_jar_cmd >> pig_cmd >> join
-
- # [START howto_operator_qubole_run_presto_query]
- presto_cmd = QuboleOperator(task_id="presto_cmd", command_type="prestocmd", query="show tables")
- # [END howto_operator_qubole_run_presto_query]
-
- # [START howto_operator_qubole_run_shell_script]
- shell_cmd = QuboleOperator(
- task_id="shell_cmd",
- command_type="shellcmd",
- script_location="s3://public-qubole/qbol-library/scripts/shellx.sh",
- parameters="param1 param2",
- )
- # [END howto_operator_qubole_run_shell_script]
-
- branching >> presto_cmd >> shell_cmd >> join
-
- # [START howto_operator_qubole_run_db_tap_query]
- db_query = QuboleOperator(
- task_id="db_query", command_type="dbtapquerycmd", query="show tables", db_tap_id=2064
- )
- # [END howto_operator_qubole_run_db_tap_query]
-
- # [START howto_operator_qubole_run_db_export]
- db_export = QuboleOperator(
- task_id="db_export",
- command_type="dbexportcmd",
- mode=1,
- hive_table="default_qubole_airline_origin_destination",
- db_table="exported_airline_origin_destination",
- partition_spec="dt=20110104-02",
- dbtap_id=2064,
- )
- # [END howto_operator_qubole_run_db_export]
-
- branching >> db_query >> db_export >> join
-
- # [START howto_operator_qubole_run_db_import]
- db_import = QuboleOperator(
- task_id="db_import",
- command_type="dbimportcmd",
- mode=1,
- hive_table="default_qubole_airline_origin_destination",
- db_table="exported_airline_origin_destination",
- where_clause="id < 10",
- parallelism=2,
- dbtap_id=2064,
- )
- # [END howto_operator_qubole_run_db_import]
-
- # [START howto_operator_qubole_run_spark_scala]
- prog = """
- import scala.math.random
- import org.apache.spark._
-
- /** Computes an approximation to pi */
- object SparkPi {
- def main(args: Array[String]) {
- val conf = new SparkConf().setAppName("Spark Pi")
- val spark = new SparkContext(conf)
- val slices = if (args.length > 0) args(0).toInt else 2
- val n = math.min(100000L * slices, Int.MaxValue).toInt // avoid overflow
- val count = spark.parallelize(1 until n, slices).map { i =>
- val x = random * 2 - 1
- val y = random * 2 - 1
- if (x*x + y*y < 1) 1 else 0
- }.reduce(_ + _)
- println("Pi is roughly " + 4.0 * count / n)
- spark.stop()
- }
- }
- """
-
- spark_cmd = QuboleOperator(
- task_id="spark_cmd",
- command_type="sparkcmd",
- program=prog,
- language="scala",
- arguments="--class SparkPi",
- tags="airflow_example_run",
- )
- # [END howto_operator_qubole_run_spark_scala]
-
- branching >> db_import >> spark_cmd >> join
-
- from tests.system.utils.watcher import watcher
-
- # This test needs watcher in order to properly mark success/failure
- # when "tearDown" task with trigger rule is part of the DAG
- list(dag.tasks) >> watcher()
-
-
-from tests.system.utils import get_test_run # noqa: E402
-
-# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
-test_run = get_test_run(dag)
diff --git a/tests/system/providers/qubole/example_qubole_sensors.py b/tests/system/providers/qubole/example_qubole_sensors.py
deleted file mode 100644
index d600973dff..0000000000
--- a/tests/system/providers/qubole/example_qubole_sensors.py
+++ /dev/null
@@ -1,94 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-import os
-import textwrap
-from datetime import datetime
-
-from airflow import DAG
-from airflow.providers.qubole.sensors.qubole import QuboleFileSensor, QubolePartitionSensor
-
-START_DATE = datetime(2021, 1, 1)
-
-ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
-DAG_ID = "example_qubole_sensor"
-
-
-with DAG(
- dag_id=DAG_ID,
- schedule=None,
- start_date=START_DATE,
- tags=["example"],
-) as dag:
- dag.doc_md = textwrap.dedent(
- """
- This is only an example DAG to highlight usage of QuboleSensor in various scenarios,
- some of these tasks may or may not work based on your QDS account setup.
-
- Run a shell command from Qubole Analyze against your Airflow cluster with following to
- trigger it manually `airflow dags trigger example_qubole_sensor`.
-
- *Note: Make sure that connection `qubole_default` is properly set before running
- this example.*
- """
- )
-
- # [START howto_sensor_qubole_run_file_sensor]
- check_s3_file = QuboleFileSensor(
- task_id="check_s3_file",
- poke_interval=60,
- timeout=600,
- data={
- "files": [
- "s3://paid-qubole/HadoopAPIExamples/jars/hadoop-0.20.1-dev-streaming.jar",
- "s3://paid-qubole/HadoopAPITests/data/{{ ds.split('-')[2] }}.tsv",
- ] # will check for availability of all the files in array
- },
- )
- # [END howto_sensor_qubole_run_file_sensor]
-
- # [START howto_sensor_qubole_run_partition_sensor]
- check_hive_partition = QubolePartitionSensor(
- task_id="check_hive_partition",
- poke_interval=10,
- timeout=60,
- data={
- "schema": "default",
- "table": "my_partitioned_table",
- "columns": [
- {"column": "month", "values": ["{{ ds.split('-')[1] }}"]},
- {"column": "day", "values": ["{{ ds.split('-')[2] }}", "{{ yesterday_ds.split('-')[2] }}"]},
- ], # will check for partitions like [month=12/day=12,month=12/day=13]
- },
- )
- # [END howto_sensor_qubole_run_partition_sensor]
-
- check_s3_file >> check_hive_partition
-
- from tests.system.utils.watcher import watcher
-
- # This test needs watcher in order to properly mark success/failure
- # when "tearDown" task with trigger rule is part of the DAG
- list(dag.tasks) >> watcher()
-
-
-from tests.system.utils import get_test_run # noqa: E402
-
-# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
-test_run = get_test_run(dag)