You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2021/08/02 16:47:35 UTC
[airflow] branch v2-1-test updated (3d2a5cf -> ac832a2)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a change to branch v2-1-test
in repository https://gitbox.apache.org/repos/asf/airflow.git.
from 3d2a5cf Switches to "/" convention in ghcr.io images with optimisations
new 88e6305 Switch back http provider after requests removes LGPL dependency (#16974)
new e8058eb Remove SQLAlchemy <1.4 constraint (#16630)
new c967d29 Bump Jinja2 upper-bound from 2.12.0 to 4.0.0 (#16595)
new f3b74d8 Add type annotations to setup.py (#16658)
new ab46af9 bump dnspython (#16698)
new 4aef457 AIRFLOW-5529 Add Apache Drill provider. (#16884)
new f14860d Update alias for field_mask in Google Memmcache (#16975)
new ac832a2 Updates to FlaskAppBuilder 3.3.2+ (#17208)
The 8 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CONTRIBUTING.rst | 22 ++---
INSTALL | 24 +++---
.../{asana => apache/drill}/CHANGELOG.rst | 1 +
.../providers/apache/drill}/__init__.py | 0
.../example_dags/example_drill_dag.py} | 34 ++++----
.../providers/apache/drill/hooks}/__init__.py | 0
airflow/providers/apache/drill/hooks/drill.py | 89 +++++++++++++++++++++
.../providers/apache/drill/operators}/__init__.py | 0
.../jdbc.py => apache/drill/operators/drill.py} | 47 +++++------
.../apache/{sqoop => drill}/provider.yaml | 26 +++---
.../google/cloud/hooks/cloud_memorystore.py | 16 ++--
airflow/ui/src/views/Docs.tsx | 1 +
airflow/utils/db.py | 10 +++
.../commits.rst} | 12 +--
.../connections/drill.rst} | 35 +++-----
.../index.rst | 41 +++-------
.../operators.rst} | 38 ++++-----
docs/apache-airflow/extra-packages-ref.rst | 4 +-
docs/conf.py | 1 +
docs/integration-logos/apache/drill.png | Bin 0 -> 40173 bytes
docs/spelling_wordlist.txt | 1 +
setup.cfg | 7 +-
setup.py | 72 ++++++++++-------
.../apache/drill}/__init__.py | 0
.../apache/drill/hooks}/__init__.py | 0
tests/providers/apache/drill/hooks/test_drill.py | 84 +++++++++++++++++++
.../apache/drill/operators}/__init__.py | 0
.../drill/operators/test_drill.py} | 52 ++++++------
28 files changed, 397 insertions(+), 220 deletions(-)
copy airflow/providers/{asana => apache/drill}/CHANGELOG.rst (99%)
copy {tests/www/api/experimental => airflow/providers/apache/drill}/__init__.py (100%)
copy airflow/providers/apache/{pig/example_dags/example_pig.py => drill/example_dags/example_drill_dag.py} (60%)
copy {tests/www/api/experimental => airflow/providers/apache/drill/hooks}/__init__.py (100%)
create mode 100644 airflow/providers/apache/drill/hooks/drill.py
copy {tests/www/api/experimental => airflow/providers/apache/drill/operators}/__init__.py (100%)
copy airflow/providers/{jdbc/operators/jdbc.py => apache/drill/operators/drill.py} (57%)
copy airflow/providers/apache/{sqoop => drill}/provider.yaml (63%)
copy docs/{apache-airflow-providers-amazon/secrets-backends/index.rst => apache-airflow-providers-apache-drill/commits.rst} (84%)
copy docs/{apache-airflow-providers-slack/connections/slack.rst => apache-airflow-providers-apache-drill/connections/drill.rst} (54%)
copy docs/{apache-airflow-providers-openfaas => apache-airflow-providers-apache-drill}/index.rst (62%)
copy docs/{apache-airflow-providers-google/operators/transfer/salesforce_to_gcs.rst => apache-airflow-providers-apache-drill/operators.rst} (53%)
create mode 100644 docs/integration-logos/apache/drill.png
copy tests/{www/api/experimental => providers/apache/drill}/__init__.py (100%)
copy tests/{www/api/experimental => providers/apache/drill/hooks}/__init__.py (100%)
create mode 100644 tests/providers/apache/drill/hooks/test_drill.py
copy tests/{www/api/experimental => providers/apache/drill/operators}/__init__.py (100%)
copy tests/providers/{neo4j/operators/test_neo4j.py => apache/drill/operators/test_drill.py} (52%)
[airflow] 04/08: Add type annotations to setup.py (#16658)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-1-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit f3b74d802a1bd61060a302b51b894d178b543e03
Author: Ali Muhammad <al...@gmail.com>
AuthorDate: Fri Jun 25 22:25:52 2021 +0500
Add type annotations to setup.py (#16658)
(cherry picked from commit 402274641168f412f44c545c34f3e7edf5cf1476)
---
setup.py | 40 ++++++++++++++++++++++++----------------
1 file changed, 24 insertions(+), 16 deletions(-)
diff --git a/setup.py b/setup.py
index 21097e6..e6c1d46 100644
--- a/setup.py
+++ b/setup.py
@@ -62,14 +62,14 @@ class CleanCommand(Command):
description = "Tidy up the project root"
user_options: List[str] = []
- def initialize_options(self):
+ def initialize_options(self) -> None:
"""Set default values for options."""
- def finalize_options(self):
+ def finalize_options(self) -> None:
"""Set final values for options."""
@staticmethod
- def rm_all_files(files: List[str]):
+ def rm_all_files(files: List[str]) -> None:
"""Remove all files from the list"""
for file in files:
try:
@@ -77,7 +77,7 @@ class CleanCommand(Command):
except Exception as e:
logger.warning("Error when removing %s: %s", file, e)
- def run(self):
+ def run(self) -> None:
"""Remove temporary files and directories."""
os.chdir(my_dir)
self.rm_all_files(glob.glob('./build/*'))
@@ -98,10 +98,10 @@ class CompileAssets(Command):
description = "Compile and build the frontend assets"
user_options: List[str] = []
- def initialize_options(self):
+ def initialize_options(self) -> None:
"""Set default values for options."""
- def finalize_options(self):
+ def finalize_options(self) -> None:
"""Set final values for options."""
def run(self) -> None:
@@ -118,10 +118,10 @@ class ListExtras(Command):
description = "List available extras"
user_options: List[str] = []
- def initialize_options(self):
+ def initialize_options(self) -> None:
"""Set default values for options."""
- def finalize_options(self):
+ def finalize_options(self) -> None:
"""Set final values for options."""
def run(self) -> None:
@@ -165,7 +165,7 @@ def git_version(version_: str) -> str:
return 'no_git_version'
-def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version"])):
+def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version"])) -> None:
"""
Write the Semver version + git hash to file, e.g. ".dev0+2f635dc265e78db6708f59f68e8009abb92c1e65".
@@ -766,7 +766,7 @@ PACKAGES_EXCLUDED_FOR_ALL.extend(
)
-def is_package_excluded(package: str, exclusion_list: List[str]):
+def is_package_excluded(package: str, exclusion_list: List[str]) -> bool:
"""
Checks if package should be excluded.
@@ -820,7 +820,7 @@ PREINSTALLED_PROVIDERS = [
]
-def get_provider_package_from_package_id(package_id: str):
+def get_provider_package_from_package_id(package_id: str) -> str:
"""
Builds the name of provider package out of the package id provided/
@@ -831,16 +831,18 @@ def get_provider_package_from_package_id(package_id: str):
return f"apache-airflow-providers-{package_suffix}"
-def get_excluded_providers():
+def get_excluded_providers() -> List[str]:
"""
Returns packages excluded for the current python version.
+
Currently the only excluded provider is apache hive for Python 3.9.
Until https://github.com/dropbox/PyHive/issues/380 is fixed.
+
"""
return ['apache.hive'] if PY39 else []
-def get_all_provider_packages():
+def get_all_provider_packages() -> str:
"""Returns all provider packages configured in setup.py"""
excluded_providers = get_excluded_providers()
return " ".join(
@@ -851,7 +853,13 @@ def get_all_provider_packages():
class AirflowDistribution(Distribution):
- """The setuptools.Distribution subclass with Airflow specific behaviour"""
+ """
+ The setuptools.Distribution subclass with Airflow specific behaviour
+
+ The reason for pylint: disable=signature-differs of parse_config_files is explained here:
+ https://github.com/PyCQA/pylint/issues/3737
+
+ """
def parse_config_files(self, *args, **kwargs) -> None:
"""
@@ -949,7 +957,7 @@ def add_all_provider_packages() -> None:
class Develop(develop_orig):
"""Forces removal of providers in editable mode."""
- def run(self):
+ def run(self) -> None:
self.announce('Installing in editable mode. Uninstalling provider packages!', level=log.INFO)
# We need to run "python3 -m pip" because it might be that older PIP binary is in the path
# And it results with an error when running pip directly (cannot import pip module)
@@ -973,7 +981,7 @@ class Develop(develop_orig):
class Install(install_orig):
"""Forces installation of providers from sources in editable mode."""
- def run(self):
+ def run(self) -> None:
self.announce('Standard installation. Providers are installed from packages', level=log.INFO)
super().run()
[airflow] 03/08: Bump Jinja2 upper-bound from 2.12.0 to 4.0.0
(#16595)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-1-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit c967d29671b854fdad0d8eba11a807bc3bf3ce3c
Author: Ashwin Madavan <as...@gmail.com>
AuthorDate: Thu Jun 24 15:07:23 2021 -0400
Bump Jinja2 upper-bound from 2.12.0 to 4.0.0 (#16595)
(cherry picked from commit 5d5268f5e553a7031ebfb08754c31fca5c13bda7)
---
setup.cfg | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/setup.cfg b/setup.cfg
index 0e4868f..fbe58cb 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -114,7 +114,7 @@ install_requires =
iso8601>=0.1.12
# Logging is broken with itsdangerous > 2
itsdangerous>=1.1.0, <2.0
- jinja2>=2.10.1, <2.12.0
+ jinja2>=2.10.1,<4
jsonschema~=3.0
lazy-object-proxy
lockfile>=0.12.2
[airflow] 01/08: Switch back http provider after requests removes
LGPL dependency (#16974)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-1-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 88e6305c92a3e96134589f3bfb29b3ff9803fafd
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Tue Jul 13 22:13:30 2021 +0200
Switch back http provider after requests removes LGPL dependency (#16974)
Following merging the https://github.com/psf/requests/pull/5797
and requests 2.26.0 release without LGPL chardet dependency,
we can now bring back http as pre-installed provider as it does
not bring chardet automatically any more.
(cherry picked from commit c46e841519ef2df7dc40ff2596dd49c010514d87)
---
docs/apache-airflow/extra-packages-ref.rst | 2 +-
setup.py | 13 ++++++-------
2 files changed, 7 insertions(+), 8 deletions(-)
diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst
index b4b4bb4..b1dff07 100644
--- a/docs/apache-airflow/extra-packages-ref.rst
+++ b/docs/apache-airflow/extra-packages-ref.rst
@@ -258,7 +258,7 @@ Those are extras that provide support for integration with external systems via
+---------------------+-----------------------------------------------------+--------------------------------------+--------------+
| grpc | ``pip install 'apache-airflow[grpc]'`` | Grpc hooks and operators | |
+---------------------+-----------------------------------------------------+--------------------------------------+--------------+
-| http | ``pip install 'apache-airflow[http]'`` | HTTP hooks, operators and sensors | |
+| http | ``pip install 'apache-airflow[http]'`` | HTTP hooks, operators and sensors | * |
+---------------------+-----------------------------------------------------+--------------------------------------+--------------+
| imap | ``pip install 'apache-airflow[imap]'`` | IMAP hooks and sensors | * |
+---------------------+-----------------------------------------------------+--------------------------------------+--------------+
diff --git a/setup.py b/setup.py
index 5d6f752..21097e6 100644
--- a/setup.py
+++ b/setup.py
@@ -231,13 +231,13 @@ dask = [
'distributed>=2.11.1, <2.20',
]
databricks = [
- 'requests>=2.20.0, <3',
+ 'requests>=2.26.0, <3',
]
datadog = [
'datadog>=0.14.0',
]
deprecated_api = [
- 'requests>=2.20.0',
+ 'requests>=2.26.0',
]
doc = [
# Sphinx is limited to < 3.5.0 because of https://github.com/sphinx-doc/sphinx/issues/8880
@@ -330,7 +330,9 @@ hive = [
'thrift>=0.9.2',
]
http = [
- 'requests>=2.20.0',
+ # The 2.26.0 release of requests got rid of the chardet LGPL mandatory dependency, allowing us to
+ # release it as a requirement for airflow
+ 'requests>=2.26.0',
]
http_provider = [
# NOTE ! The HTTP provider is NOT preinstalled by default when Airflow is installed - because it
@@ -810,12 +812,9 @@ EXTRAS_REQUIREMENTS = sort_extras_requirements()
# Those providers are pre-installed always when airflow is installed.
# Those providers do not have dependency on airflow2.0 because that would lead to circular dependencies.
# This is not a problem for PIP but some tools (pipdeptree) show those as a warning.
-# NOTE ! The HTTP provider is NOT preinstalled by default when Airflow is installed - because it
-# depends on `requests` library and until `chardet` is mandatory dependency of `requests`
-# we cannot make it mandatory dependency. See https://github.com/psf/requests/pull/5797
PREINSTALLED_PROVIDERS = [
'ftp',
- # 'http',
+ 'http',
'imap',
'sqlite',
]
[airflow] 08/08: Updates to FlaskAppBuilder 3.3.2+ (#17208)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-1-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit ac832a2f8446d5a554af2b9833453d8a8bbcb195
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Wed Jul 28 21:37:48 2021 +0200
Updates to FlaskAppBuilder 3.3.2+ (#17208)
There are some clarifications about using the authentication
via FlaskAppBuilder - the change implements minimum version of the
FAB to 3.3.2 and clarifies the dependencies used in FAB 3 series
to be only authlib rather than flask-oauth.
Fixes: #16944 (this is the second, proper fix this time).
(cherry picked from commit 6d7fa874ff201af7f602be9c58a827998814bdd1)
---
setup.cfg | 2 +-
setup.py | 10 ++++------
2 files changed, 5 insertions(+), 7 deletions(-)
diff --git a/setup.cfg b/setup.cfg
index fbe58cb..d3c5f57 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -100,7 +100,7 @@ install_requires =
# https://github.com/readthedocs/sphinx_rtd_theme/issues/1115
docutils<0.17
flask>=1.1.0, <2.0
- flask-appbuilder~=3.3
+ flask-appbuilder>=3.3.2, <4.0.0
flask-caching>=1.5.0, <2.0.0
flask-login>=0.3, <0.5
flask-wtf>=0.14.3, <0.15
diff --git a/setup.py b/setup.py
index 46ff73d..c74808a 100644
--- a/setup.py
+++ b/setup.py
@@ -270,10 +270,8 @@ exasol = [
facebook = [
'facebook-business>=6.0.2',
]
-flask_oauth = [
- 'Flask-OAuthlib>=0.9.1,<0.9.6', # Flask OAuthLib 0.9.6 requires Flask-Login 0.5.0 - breaks FAB
- 'oauthlib!=2.0.3,!=2.0.4,!=2.0.5,<3.0.0,>=1.1.2',
- 'requests-oauthlib<1.2.0',
+flask_appbuilder_authlib = [
+ 'authlib',
]
google = [
'PyOpenSSL',
@@ -622,8 +620,8 @@ CORE_EXTRAS_REQUIREMENTS: Dict[str, List[str]] = {
'cncf.kubernetes': kubernetes, # also has provider, but it extends the core with the KubernetesExecutor
'dask': dask,
'deprecated_api': deprecated_api,
- 'github_enterprise': flask_oauth,
- 'google_auth': flask_oauth,
+ 'github_enterprise': flask_appbuilder_authlib,
+ 'google_auth': flask_appbuilder_authlib,
'kerberos': kerberos,
'ldap': ldap,
'leveldb': leveldb,
[airflow] 05/08: bump dnspython (#16698)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-1-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit ab46af97bd50ad0f024892eb1c65d7bbb724ae89
Author: kurtqq <47...@users.noreply.github.com>
AuthorDate: Tue Jun 29 00:21:24 2021 +0300
bump dnspython (#16698)
(cherry picked from commit 57dcac22137bc958c1ed9f12fa54484e13411a6f)
---
setup.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/setup.py b/setup.py
index e6c1d46..9e8c28d 100644
--- a/setup.py
+++ b/setup.py
@@ -370,7 +370,7 @@ ldap = [
]
leveldb = ['plyvel']
mongo = [
- 'dnspython>=1.13.0,<2.0.0',
+ 'dnspython>=1.13.0,<3.0.0',
'pymongo>=3.6.0',
]
mssql = [
[airflow] 02/08: Remove SQLAlchemy <1.4 constraint (#16630)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-1-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit e8058ebdc6df5d7a168dcccf4d8042ec09faac80
Author: Tzu-ping Chung <tp...@astronomer.io>
AuthorDate: Thu Jun 24 21:10:15 2021 +0800
Remove SQLAlchemy <1.4 constraint (#16630)
This was added due to flask-sqlalchemy and sqlalchemy-utils not declaring
the upper bounds. They have since released sqlalchemy 1.4-compatible
versions, so we can remove that hack.
Note that this does *not* actually make us run on sqlalchemy 1.4 since
flask-appbuilder still has a <1.4 pin. But that's for flask-appbuilder
to worry about -- code in Airflow is compatible, so we can remove the
constraint now, and get sqlalchemy 1.4 as soon as flask-appbuilder
allows us to.
(cherry picked from commit d181604739c048c6969d8997dbaf8b159607904b)
---
setup.cfg | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/setup.cfg b/setup.cfg
index 8b03296..0e4868f 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -150,8 +150,7 @@ install_requires =
pyyaml>=5.1
rich>=9.2.0
setproctitle>=1.1.8, <2
- # SQLAlchemy 1.4 breaks sqlalchemy-utils https://github.com/kvesteri/sqlalchemy-utils/issues/505
- sqlalchemy>=1.3.18, <1.4
+ sqlalchemy>=1.3.18
sqlalchemy_jsonfield~=1.0
# Required by vendored-in connexion
swagger-ui-bundle>=0.0.2
[airflow] 07/08: Update alias for field_mask in Google Memmcache
(#16975)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-1-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit f14860d6a7aefe8f98bf6eeba648c487b9cee2cf
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Tue Jul 13 20:54:38 2021 +0200
Update alias for field_mask in Google Memmcache (#16975)
The July 12 2021 release of google-memcache library removed
field_mask alias from the library which broke our typechecking
and made google provider unimportable. This PR fixes the import
to use the actual import.
(cherry picked from commit a3f5c93806258b5ad396a638ba0169eca7f9d065)
---
.../providers/google/cloud/hooks/cloud_memorystore.py | 16 ++++++++--------
setup.py | 4 +++-
2 files changed, 11 insertions(+), 9 deletions(-)
diff --git a/airflow/providers/google/cloud/hooks/cloud_memorystore.py b/airflow/providers/google/cloud/hooks/cloud_memorystore.py
index caf1cd6..8f4165b 100644
--- a/airflow/providers/google/cloud/hooks/cloud_memorystore.py
+++ b/airflow/providers/google/cloud/hooks/cloud_memorystore.py
@@ -487,8 +487,8 @@ class CloudMemorystoreHook(GoogleBaseHook):
- ``redisConfig``
If a dict is provided, it must be of the same form as the protobuf message
- :class:`~google.cloud.redis_v1.types.FieldMask`
- :type update_mask: Union[Dict, google.cloud.redis_v1.types.FieldMask]
+ :class:`~google.protobuf.field_mask_pb2.FieldMask`
+ :type update_mask: Union[Dict, google.protobuf.field_mask_pb2.FieldMask]
:param instance: Required. Update description. Only fields specified in ``update_mask`` are updated.
If a dict is provided, it must be of the same form as the protobuf message
@@ -871,7 +871,7 @@ class CloudMemorystoreMemcachedHook(GoogleBaseHook):
@GoogleBaseHook.fallback_to_default_project_id
def update_instance(
self,
- update_mask: Union[Dict, cloud_memcache.field_mask.FieldMask],
+ update_mask: Union[Dict, FieldMask],
instance: Union[Dict, cloud_memcache.Instance],
project_id: str,
location: Optional[str] = None,
@@ -889,9 +889,9 @@ class CloudMemorystoreMemcachedHook(GoogleBaseHook):
- ``displayName``
If a dict is provided, it must be of the same form as the protobuf message
- :class:`~google.cloud.memcache_v1beta2.types.cloud_memcache.field_mask.FieldMask`
+ :class:`~google.protobuf.field_mask_pb2.FieldMask`)
:type update_mask:
- Union[Dict, google.cloud.memcache_v1beta2.types.cloud_memcache.field_mask.FieldMask]
+ Union[Dict, google.protobuf.field_mask_pb2.FieldMask]
:param instance: Required. Update description. Only fields specified in ``update_mask`` are updated.
If a dict is provided, it must be of the same form as the protobuf message
@@ -935,7 +935,7 @@ class CloudMemorystoreMemcachedHook(GoogleBaseHook):
@GoogleBaseHook.fallback_to_default_project_id
def update_parameters(
self,
- update_mask: Union[Dict, cloud_memcache.field_mask.FieldMask],
+ update_mask: Union[Dict, FieldMask],
parameters: Union[Dict, cloud_memcache.MemcacheParameters],
project_id: str,
location: str,
@@ -951,9 +951,9 @@ class CloudMemorystoreMemcachedHook(GoogleBaseHook):
:param update_mask: Required. Mask of fields to update.
If a dict is provided, it must be of the same form as the protobuf message
- :class:`~google.cloud.memcache_v1beta2.types.cloud_memcache.field_mask.FieldMask`
+ :class:`~google.protobuf.field_mask_pb2.FieldMask`
:type update_mask:
- Union[Dict, google.cloud.memcache_v1beta2.types.cloud_memcache.field_mask.FieldMask]
+ Union[Dict, google.protobuf.field_mask_pb2.FieldMask]
:param parameters: The parameters to apply to the instance.
If a dict is provided, it must be of the same form as the protobuf message
:class:`~google.cloud.memcache_v1beta2.types.cloud_memcache.MemcacheParameters`
diff --git a/setup.py b/setup.py
index 9dde824..46ff73d 100644
--- a/setup.py
+++ b/setup.py
@@ -292,7 +292,9 @@ google = [
'google-cloud-kms>=2.0.0,<3.0.0',
'google-cloud-language>=1.1.1,<2.0.0',
'google-cloud-logging>=2.1.1,<3.0.0',
- 'google-cloud-memcache>=0.2.0',
+ # 1.1.0 removed field_mask and broke import for released providers
+ # We can remove the <1.1.0 limitation after we release new Google Provider
+ 'google-cloud-memcache>=0.2.0,<1.1.0',
'google-cloud-monitoring>=2.0.0,<3.0.0',
'google-cloud-os-login>=2.0.0,<3.0.0',
'google-cloud-pubsub>=2.0.0,<3.0.0',
[airflow] 06/08: AIRFLOW-5529 Add Apache Drill provider. (#16884)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-1-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 4aef457774c646940caef7a02e0c2c0d76915160
Author: dzamo <91...@users.noreply.github.com>
AuthorDate: Mon Jul 12 19:59:35 2021 +0200
AIRFLOW-5529 Add Apache Drill provider. (#16884)
(cherry picked from commit 8808b641942e1b81c21db054fd6d36e2031cfab8)
---
CONTRIBUTING.rst | 22 ++---
INSTALL | 24 +++---
airflow/providers/apache/drill/CHANGELOG.rst | 25 ++++++
airflow/providers/apache/drill/__init__.py | 17 ++++
.../apache/drill/example_dags/example_drill_dag.py | 46 +++++++++++
airflow/providers/apache/drill/hooks/__init__.py | 17 ++++
airflow/providers/apache/drill/hooks/drill.py | 89 +++++++++++++++++++++
.../providers/apache/drill/operators/__init__.py | 17 ++++
airflow/providers/apache/drill/operators/drill.py | 71 ++++++++++++++++
airflow/providers/apache/drill/provider.yaml | 49 ++++++++++++
airflow/ui/src/views/Docs.tsx | 1 +
airflow/utils/db.py | 10 +++
.../commits.rst | 23 ++++++
.../connections/drill.rst | 44 ++++++++++
.../index.rst | 50 ++++++++++++
.../operators.rst | 51 ++++++++++++
docs/apache-airflow/extra-packages-ref.rst | 2 +
docs/conf.py | 1 +
docs/integration-logos/apache/drill.png | Bin 0 -> 40173 bytes
docs/spelling_wordlist.txt | 1 +
setup.py | 3 +
tests/providers/apache/drill/__init__.py | 17 ++++
tests/providers/apache/drill/hooks/__init__.py | 17 ++++
tests/providers/apache/drill/hooks/test_drill.py | 84 +++++++++++++++++++
tests/providers/apache/drill/operators/__init__.py | 17 ++++
.../providers/apache/drill/operators/test_drill.py | 63 +++++++++++++++
26 files changed, 738 insertions(+), 23 deletions(-)
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index 98cdf93..be807f4 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -573,17 +573,17 @@ This is the full list of those extras:
.. START EXTRAS HERE
-airbyte, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.druid,
-apache.hdfs, apache.hive, apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark,
-apache.sqoop, apache.webhdfs, asana, async, atlas, aws, azure, cassandra, celery, cgroups, cloudant,
-cncf.kubernetes, crypto, dask, databricks, datadog, deprecated_api, devel, devel_all, devel_ci,
-devel_hadoop, dingding, discord, doc, docker, druid, elasticsearch, exasol, facebook, ftp, gcp,
-gcp_api, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, imap, jdbc,
-jenkins, jira, kerberos, kubernetes, ldap, leveldb, microsoft.azure, microsoft.mssql,
-microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openfaas, opsgenie, oracle, pagerduty, papermill,
-password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba,
-segment, sendgrid, sentry, sftp, singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau,
-telegram, trino, vertica, virtualenv, webhdfs, winrm, yandex, zendesk
+airbyte, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.drill,
+apache.druid, apache.hdfs, apache.hive, apache.kylin, apache.livy, apache.pig, apache.pinot,
+apache.spark, apache.sqoop, apache.webhdfs, asana, async, atlas, aws, azure, cassandra, celery,
+cgroups, cloudant, cncf.kubernetes, crypto, dask, databricks, datadog, deprecated_api, devel,
+devel_all, devel_ci, devel_hadoop, dingding, discord, doc, docker, druid, elasticsearch, exasol,
+facebook, ftp, gcp, gcp_api, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive,
+http, imap, jdbc, jenkins, jira, kerberos, kubernetes, ldap, leveldb, microsoft.azure,
+microsoft.mssql, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openfaas, opsgenie, oracle,
+pagerduty, papermill, password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3,
+salesforce, samba, segment, sendgrid, sentry, sftp, singularity, slack, snowflake, spark, sqlite,
+ssh, statsd, tableau, telegram, trino, vertica, virtualenv, webhdfs, winrm, yandex, zendesk
.. END EXTRAS HERE
diff --git a/INSTALL b/INSTALL
index 111b51f..554af5c 100644
--- a/INSTALL
+++ b/INSTALL
@@ -1,6 +1,6 @@
# INSTALL / BUILD instructions for Apache Airflow
-This ia a generic installation method that requires a number of dependencies to be installed.
+This is a generic installation method that requires a number of dependencies to be installed.
Depending on your system you might need different prerequisites, but the following
systems/prerequisites are known to work:
@@ -89,17 +89,17 @@ The list of available extras:
# START EXTRAS HERE
-airbyte, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.druid,
-apache.hdfs, apache.hive, apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark,
-apache.sqoop, apache.webhdfs, asana, async, atlas, aws, azure, cassandra, celery, cgroups, cloudant,
-cncf.kubernetes, crypto, dask, databricks, datadog, deprecated_api, devel, devel_all, devel_ci,
-devel_hadoop, dingding, discord, doc, docker, druid, elasticsearch, exasol, facebook, ftp, gcp,
-gcp_api, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, imap, jdbc,
-jenkins, jira, kerberos, kubernetes, ldap, leveldb, microsoft.azure, microsoft.mssql,
-microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openfaas, opsgenie, oracle, pagerduty, papermill,
-password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba,
-segment, sendgrid, sentry, sftp, singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau,
-telegram, trino, vertica, virtualenv, webhdfs, winrm, yandex, zendesk
+airbyte, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.drill,
+apache.druid, apache.hdfs, apache.hive, apache.kylin, apache.livy, apache.pig, apache.pinot,
+apache.spark, apache.sqoop, apache.webhdfs, asana, async, atlas, aws, azure, cassandra, celery,
+cgroups, cloudant, cncf.kubernetes, crypto, dask, databricks, datadog, deprecated_api, devel,
+devel_all, devel_ci, devel_hadoop, dingding, discord, doc, docker, druid, elasticsearch, exasol,
+facebook, ftp, gcp, gcp_api, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive,
+http, imap, jdbc, jenkins, jira, kerberos, kubernetes, ldap, leveldb, microsoft.azure,
+microsoft.mssql, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openfaas, opsgenie, oracle,
+pagerduty, papermill, password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3,
+salesforce, samba, segment, sendgrid, sentry, sftp, singularity, slack, snowflake, spark, sqlite,
+ssh, statsd, tableau, telegram, trino, vertica, virtualenv, webhdfs, winrm, yandex, zendesk
# END EXTRAS HERE
diff --git a/airflow/providers/apache/drill/CHANGELOG.rst b/airflow/providers/apache/drill/CHANGELOG.rst
new file mode 100644
index 0000000..cef7dda
--- /dev/null
+++ b/airflow/providers/apache/drill/CHANGELOG.rst
@@ -0,0 +1,25 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+
+Changelog
+---------
+
+1.0.0
+.....
+
+Initial version of the provider.
diff --git a/airflow/providers/apache/drill/__init__.py b/airflow/providers/apache/drill/__init__.py
new file mode 100644
index 0000000..217e5db
--- /dev/null
+++ b/airflow/providers/apache/drill/__init__.py
@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/airflow/providers/apache/drill/example_dags/example_drill_dag.py b/airflow/providers/apache/drill/example_dags/example_drill_dag.py
new file mode 100644
index 0000000..60a35ee
--- /dev/null
+++ b/airflow/providers/apache/drill/example_dags/example_drill_dag.py
@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Example Airflow DAG to submit Apache Spark applications using
+`SparkSubmitOperator`, `SparkJDBCOperator` and `SparkSqlOperator`.
+"""
+from airflow.models import DAG
+from airflow.providers.apache.drill.operators.drill import DrillOperator
+from airflow.utils.dates import days_ago
+
+args = {
+ 'owner': 'Airflow',
+}
+
+with DAG(
+ dag_id='example_drill_dag',
+ default_args=args,
+ schedule_interval=None,
+ start_date=days_ago(2),
+ tags=['example'],
+) as dag:
+ # [START howto_operator_drill]
+ sql_task = DrillOperator(
+ task_id='json_to_parquet_table',
+ sql='''
+ drop table if exists dfs.tmp.employee;
+ create table dfs.tmp.employee as select * from cp.`employee.json`;
+ ''',
+ )
+ # [END howto_operator_drill]
diff --git a/airflow/providers/apache/drill/hooks/__init__.py b/airflow/providers/apache/drill/hooks/__init__.py
new file mode 100644
index 0000000..217e5db
--- /dev/null
+++ b/airflow/providers/apache/drill/hooks/__init__.py
@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/airflow/providers/apache/drill/hooks/drill.py b/airflow/providers/apache/drill/hooks/drill.py
new file mode 100644
index 0000000..470be8c
--- /dev/null
+++ b/airflow/providers/apache/drill/hooks/drill.py
@@ -0,0 +1,89 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Any, Iterable, Optional, Tuple
+
+from sqlalchemy import create_engine
+from sqlalchemy.engine import Connection
+
+from airflow.hooks.dbapi import DbApiHook
+
+
+class DrillHook(DbApiHook):
+ """
+ Interact with Apache Drill via sqlalchemy-drill.
+
+ You can specify the SQLAlchemy dialect and driver that sqlalchemy-drill
+ will employ to communicate with Drill in the extras field of your
+ connection, e.g. ``{"dialect_driver": "drill+sadrill"}`` for communication
+ over Drill's REST API. See the sqlalchemy-drill documentation for
+ descriptions of the supported dialects and drivers.
+
+ You can specify the default storage_plugin for the sqlalchemy-drill
+ connection using the extras field e.g. ``{"storage_plugin": "dfs"}``.
+ """
+
+ conn_name_attr = 'drill_conn_id'
+ default_conn_name = 'drill_default'
+ conn_type = 'drill'
+ hook_name = 'Drill'
+ supports_autocommit = False
+
+ def get_conn(self) -> Connection:
+ """Establish a connection to Drillbit."""
+ conn_md = self.get_connection(getattr(self, self.conn_name_attr))
+ creds = f'{conn_md.login}:{conn_md.password}@' if conn_md.login else ''
+ engine = create_engine(
+ f'{conn_md.extra_dejson.get("dialect_driver", "drill+sadrill")}://{creds}'
+ f'{conn_md.host}:{conn_md.port}/'
+ f'{conn_md.extra_dejson.get("storage_plugin", "dfs")}'
+ )
+
+ self.log.info(
+ 'Connected to the Drillbit at %s:%s as user %s', conn_md.host, conn_md.port, conn_md.login
+ )
+ return engine.raw_connection()
+
+ def get_uri(self) -> str:
+ """
+ Returns the connection URI
+
+ e.g: ``drill://localhost:8047/dfs``
+ """
+ conn_md = self.get_connection(getattr(self, self.conn_name_attr))
+ host = conn_md.host
+ if conn_md.port is not None:
+ host += f':{conn_md.port}'
+ conn_type = 'drill' if not conn_md.conn_type else conn_md.conn_type
+ dialect_driver = conn_md.extra_dejson.get('dialect_driver', 'drill+sadrill')
+ storage_plugin = conn_md.extra_dejson.get('storage_plugin', 'dfs')
+ return f'{conn_type}://{host}/{storage_plugin}' f'?dialect_driver={dialect_driver}'
+
+ def set_autocommit(self, conn: Connection, autocommit: bool) -> NotImplemented:
+ raise NotImplementedError("There are no transactions in Drill.")
+
+ def insert_rows(
+ self,
+ table: str,
+ rows: Iterable[Tuple[str]],
+ target_fields: Optional[Iterable[str]] = None,
+ commit_every: int = 1000,
+ replace: bool = False,
+ **kwargs: Any,
+ ) -> NotImplemented:
+ raise NotImplementedError("There is no INSERT statement in Drill.")
diff --git a/airflow/providers/apache/drill/operators/__init__.py b/airflow/providers/apache/drill/operators/__init__.py
new file mode 100644
index 0000000..217e5db
--- /dev/null
+++ b/airflow/providers/apache/drill/operators/__init__.py
@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/airflow/providers/apache/drill/operators/drill.py b/airflow/providers/apache/drill/operators/drill.py
new file mode 100644
index 0000000..459c623
--- /dev/null
+++ b/airflow/providers/apache/drill/operators/drill.py
@@ -0,0 +1,71 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from typing import Iterable, Mapping, Optional, Union
+
+import sqlparse
+
+from airflow.models import BaseOperator
+from airflow.providers.apache.drill.hooks.drill import DrillHook
+from airflow.utils.decorators import apply_defaults
+
+
+class DrillOperator(BaseOperator):
+ """
+ Executes the provided SQL in the identified Drill environment.
+
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:DrillOperator`
+
+ :param sql: the SQL code to be executed. (templated)
+ :type sql: Can receive a str representing a sql statement,
+ a list of str (sql statements), or a reference to a template file.
+ Template references are recognized by str ending in '.sql'
+ :param drill_conn_id: id of the connection config for the target Drill
+ environment
+ :type drill_conn_id: str
+ :param parameters: (optional) the parameters to render the SQL query with.
+ :type parameters: dict or iterable
+ """
+
+ template_fields = ('sql',)
+ template_fields_renderers = {'sql': 'sql'}
+ template_ext = ('.sql',)
+ ui_color = '#ededed'
+
+ @apply_defaults
+ def __init__(
+ self,
+ *,
+ sql: str,
+ drill_conn_id: str = 'drill_default',
+ parameters: Optional[Union[Mapping, Iterable]] = None,
+ **kwargs,
+ ) -> None:
+ super().__init__(**kwargs)
+ self.sql = sql
+ self.drill_conn_id = drill_conn_id
+ self.parameters = parameters
+ self.hook = None
+
+ def execute(self, context):
+ self.log.info('Executing: %s on %s', self.sql, self.drill_conn_id)
+ self.hook = DrillHook(drill_conn_id=self.drill_conn_id)
+ sql = sqlparse.split(sqlparse.format(self.sql, strip_comments=True))
+ no_term_sql = [s[:-1] for s in sql if s[-1] == ';']
+ self.hook.run(no_term_sql, parameters=self.parameters)
diff --git a/airflow/providers/apache/drill/provider.yaml b/airflow/providers/apache/drill/provider.yaml
new file mode 100644
index 0000000..88021e6
--- /dev/null
+++ b/airflow/providers/apache/drill/provider.yaml
@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+package-name: apache-airflow-providers-apache-drill
+name: Apache Drill
+description: |
+ `Apache Drill <https://drill.apache.org/>`__.
+
+versions:
+ - 1.0.0
+
+additional-dependencies:
+ - apache-airflow>=2.1.0
+
+integrations:
+ - integration-name: Apache Drill
+ external-doc-url: https://drill.apache.org/
+ how-to-guide:
+ - /docs/apache-airflow-providers-apache-drill/operators.rst
+ logo: /integration-logos/apache/drill.png
+ tags: [apache]
+
+operators:
+ - integration-name: Apache Drill
+ python-modules:
+ - airflow.providers.apache.drill.operators.drill
+
+hooks:
+ - integration-name: Apache Drill
+ python-modules:
+ - airflow.providers.apache.drill.hooks.drill
+
+hook-class-names:
+ - airflow.providers.apache.drill.hooks.drill.DrillHook
diff --git a/airflow/ui/src/views/Docs.tsx b/airflow/ui/src/views/Docs.tsx
index 754b803..ea42ca6 100644
--- a/airflow/ui/src/views/Docs.tsx
+++ b/airflow/ui/src/views/Docs.tsx
@@ -41,6 +41,7 @@ const Docs: React.FC = () => {
{ path: 'amazon', name: 'Amazon' },
{ path: 'apache-beam', name: 'Apache Beam' },
{ path: 'apache-cassandra', name: 'Apache Cassandra' },
+ { path: 'apache-drill', name: 'Apache Drill' },
{ path: 'apache-druid', name: 'Apache Druid' },
{ path: 'apache-hdfs', name: 'Apache HDFS' },
{ path: 'apache-hive', name: 'Apache Hive' },
diff --git a/airflow/utils/db.py b/airflow/utils/db.py
index ae8dc0e..69e2e00 100644
--- a/airflow/utils/db.py
+++ b/airflow/utils/db.py
@@ -168,6 +168,16 @@ def create_default_connections(session=None):
)
merge_conn(
Connection(
+ conn_id="drill_default",
+ conn_type="drill",
+ host="localhost",
+ port=8047,
+ extra='{"dialect_driver": "drill+sadrill", "storage_plugin": "dfs"}',
+ ),
+ session,
+ )
+ merge_conn(
+ Connection(
conn_id="druid_broker_default",
conn_type="druid",
host="druid-broker",
diff --git a/docs/apache-airflow-providers-apache-drill/commits.rst b/docs/apache-airflow-providers-apache-drill/commits.rst
new file mode 100644
index 0000000..deb31b3
--- /dev/null
+++ b/docs/apache-airflow-providers-apache-drill/commits.rst
@@ -0,0 +1,23 @@
+
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+
+Package apache-airflow-providers-apache-drill
+------------------------------------------------------
+
+`Apache Drill <https://drill.apache.org/>`__.
diff --git a/docs/apache-airflow-providers-apache-drill/connections/drill.rst b/docs/apache-airflow-providers-apache-drill/connections/drill.rst
new file mode 100644
index 0000000..05e00a2
--- /dev/null
+++ b/docs/apache-airflow-providers-apache-drill/connections/drill.rst
@@ -0,0 +1,44 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+
+
+.. _howto/connection:drill:
+
+Apache Drill Connection
+=======================
+
+The Apache Drill connection type configures a connection to Apache Drill via the sqlalchemy-drill Python package.
+
+Default Connection IDs
+----------------------
+
+Drill hooks and operators use ``drill_default`` by default.
+
+Configuring the Connection
+--------------------------
+Host (required)
+ The host of the Drillbit to connect to (HTTP, JDBC) or the DSN of the Drill ODBC connection.
+
+Port (optional)
+ The port of the Drillbit to connect to.
+
+Extra (optional)
+ A JSON dictionary specifying the extra parameters that can be used in sqlalchemy-drill connection.
+
+ * ``dialect_driver`` - The dialect and driver as understood by sqlalchemy-drill. Defaults to ``drill_sadrill`` (HTTP).
+ * ``storage_plugin`` - The default Drill storage plugin for this connection. Defaults to ``dfs``.
diff --git a/docs/apache-airflow-providers-apache-drill/index.rst b/docs/apache-airflow-providers-apache-drill/index.rst
new file mode 100644
index 0000000..9ef0f2e
--- /dev/null
+++ b/docs/apache-airflow-providers-apache-drill/index.rst
@@ -0,0 +1,50 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+``apache-airflow-providers-apache-drill``
+=========================================
+
+Content
+-------
+
+.. toctree::
+ :maxdepth: 1
+ :caption: Guides
+
+ Connection types <connections/drill>
+ Operators <operators>
+
+.. toctree::
+ :maxdepth: 1
+ :caption: References
+
+ Python API <_api/airflow/providers/apache/drill/index>
+
+.. toctree::
+ :maxdepth: 1
+ :caption: Resources
+
+ Example DAGs <https://github.com/apache/airflow/tree/main/airflow/providers/apache/drill/example_dags>
+ PyPI Repository <https://pypi.org/project/apache-airflow-providers-apache-drill/>
+
+.. THE REMINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME!
+
+.. toctree::
+ :maxdepth: 1
+ :caption: Commits
+
+ Detailed list of commits <commits>
diff --git a/docs/apache-airflow-providers-apache-drill/operators.rst b/docs/apache-airflow-providers-apache-drill/operators.rst
new file mode 100644
index 0000000..f2d5cf5
--- /dev/null
+++ b/docs/apache-airflow-providers-apache-drill/operators.rst
@@ -0,0 +1,51 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+
+Apache Drill Operators
+======================
+
+.. contents::
+ :depth: 1
+ :local:
+
+Prerequisite
+------------
+
+To use ``DrillOperator``, you must configure a :doc:`Drill Connection <connections/drill>`.
+
+
+.. _howto/operator:DrillOperator:
+
+DrillOperator
+-------------
+
+Executes one or more SQL queries on an Apache Drill server. The ``sql`` parameter can be templated and be an external ``.sql`` file.
+
+Using the operator
+""""""""""""""""""
+
+.. exampleinclude:: /../../airflow/providers/apache/drill/example_dags/example_drill_dag.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_drill]
+ :end-before: [END howto_operator_drill]
+
+Reference
+"""""""""
+
+For further information, see `the Drill documentation on querying data <http://apache.github.io/drill/docs/query-data/>`_.
diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst
index b1dff07..2d4769e 100644
--- a/docs/apache-airflow/extra-packages-ref.rst
+++ b/docs/apache-airflow/extra-packages-ref.rst
@@ -116,6 +116,8 @@ custom bash/python providers).
+---------------------+-----------------------------------------------------+------------------------------------------------+
| apache.cassandra | ``pip install 'apache-airflow[apache.cassandra]'`` | Cassandra related operators & hooks |
+---------------------+-----------------------------------------------------+------------------------------------------------+
+| apache.drill | ``pip install 'apache-airflow[apache.drill]'`` | Drill related operators & hooks |
++---------------------+-----------------------------------------------------+------------------------------------------------+
| apache.druid | ``pip install 'apache-airflow[apache.druid]'`` | Druid related operators & hooks |
+---------------------+-----------------------------------------------------+------------------------------------------------+
| apache.hdfs | ``pip install 'apache-airflow[apache.hdfs]'`` | HDFS hooks and operators |
diff --git a/docs/conf.py b/docs/conf.py
index 3046303..da3b8e2 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -512,6 +512,7 @@ autodoc_mock_imports = [
'slack_sdk',
'smbclient',
'snowflake',
+ 'sqlalchemy-drill',
'sshtunnel',
'telegram',
'tenacity',
diff --git a/docs/integration-logos/apache/drill.png b/docs/integration-logos/apache/drill.png
new file mode 100644
index 0000000..9f76b61
Binary files /dev/null and b/docs/integration-logos/apache/drill.png differ
diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt
index c476512..7631c6c 100644
--- a/docs/spelling_wordlist.txt
+++ b/docs/spelling_wordlist.txt
@@ -120,6 +120,7 @@ Docstring
Docstrings
Dont
Driesprong
+Drillbit
Drivy
Dsn
Dynamodb
diff --git a/setup.py b/setup.py
index 9e8c28d..9dde824 100644
--- a/setup.py
+++ b/setup.py
@@ -255,6 +255,7 @@ doc = [
docker = [
'docker',
]
+drill = ['sqlalchemy-drill>=1.1.0', 'sqlparse>=0.4.1']
druid = [
'pydruid>=0.4.1',
]
@@ -534,6 +535,7 @@ PROVIDERS_REQUIREMENTS: Dict[str, List[str]] = {
'amazon': amazon,
'apache.beam': apache_beam,
'apache.cassandra': cassandra,
+ 'apache.drill': drill,
'apache.druid': druid,
'apache.hdfs': hdfs,
'apache.hive': hive,
@@ -724,6 +726,7 @@ ALL_PROVIDERS = list(PROVIDERS_REQUIREMENTS.keys())
ALL_DB_PROVIDERS = [
'apache.cassandra',
+ 'apache.drill',
'apache.druid',
'apache.hdfs',
'apache.hive',
diff --git a/tests/providers/apache/drill/__init__.py b/tests/providers/apache/drill/__init__.py
new file mode 100644
index 0000000..217e5db
--- /dev/null
+++ b/tests/providers/apache/drill/__init__.py
@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/tests/providers/apache/drill/hooks/__init__.py b/tests/providers/apache/drill/hooks/__init__.py
new file mode 100644
index 0000000..217e5db
--- /dev/null
+++ b/tests/providers/apache/drill/hooks/__init__.py
@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/tests/providers/apache/drill/hooks/test_drill.py b/tests/providers/apache/drill/hooks/test_drill.py
new file mode 100644
index 0000000..97ed71f
--- /dev/null
+++ b/tests/providers/apache/drill/hooks/test_drill.py
@@ -0,0 +1,84 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import unittest
+from unittest.mock import MagicMock
+
+from airflow.providers.apache.drill.hooks.drill import DrillHook
+
+
+class TestDrillHook(unittest.TestCase):
+ def setUp(self):
+ self.cur = MagicMock(rowcount=0)
+ self.conn = conn = MagicMock()
+ self.conn.login = 'drill_user'
+ self.conn.password = 'secret'
+ self.conn.host = 'host'
+ self.conn.port = '8047'
+ self.conn.conn_type = 'drill'
+ self.conn.extra_dejson = {'dialect_driver': 'drill+sadrill', 'storage_plugin': 'dfs'}
+ self.conn.cursor.return_value = self.cur
+
+ class TestDrillHook(DrillHook):
+ def get_conn(self):
+ return conn
+
+ def get_connection(self, conn_id):
+ return conn
+
+ self.db_hook = TestDrillHook
+
+ def test_get_uri(self):
+ db_hook = self.db_hook()
+ assert 'drill://host:8047/dfs?dialect_driver=drill+sadrill' == db_hook.get_uri()
+
+ def test_get_first_record(self):
+ statement = 'SQL'
+ result_sets = [('row1',), ('row2',)]
+ self.cur.fetchone.return_value = result_sets[0]
+
+ assert result_sets[0] == self.db_hook().get_first(statement)
+ assert self.conn.close.call_count == 1
+ assert self.cur.close.call_count == 1
+ self.cur.execute.assert_called_once_with(statement)
+
+ def test_get_records(self):
+ statement = 'SQL'
+ result_sets = [('row1',), ('row2',)]
+ self.cur.fetchall.return_value = result_sets
+
+ assert result_sets == self.db_hook().get_records(statement)
+ assert self.conn.close.call_count == 1
+ assert self.cur.close.call_count == 1
+ self.cur.execute.assert_called_once_with(statement)
+
+ def test_get_pandas_df(self):
+ statement = 'SQL'
+ column = 'col'
+ result_sets = [('row1',), ('row2',)]
+ self.cur.description = [(column,)]
+ self.cur.fetchall.return_value = result_sets
+ df = self.db_hook().get_pandas_df(statement)
+
+ assert column == df.columns[0]
+ for i in range(len(result_sets)): # pylint: disable=consider-using-enumerate
+ assert result_sets[i][0] == df.values.tolist()[i][0]
+ assert self.conn.close.call_count == 1
+ assert self.cur.close.call_count == 1
+ self.cur.execute.assert_called_once_with(statement)
diff --git a/tests/providers/apache/drill/operators/__init__.py b/tests/providers/apache/drill/operators/__init__.py
new file mode 100644
index 0000000..217e5db
--- /dev/null
+++ b/tests/providers/apache/drill/operators/__init__.py
@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/tests/providers/apache/drill/operators/test_drill.py b/tests/providers/apache/drill/operators/test_drill.py
new file mode 100644
index 0000000..3572d85
--- /dev/null
+++ b/tests/providers/apache/drill/operators/test_drill.py
@@ -0,0 +1,63 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+import pytest
+
+from airflow.models.dag import DAG
+from airflow.providers.apache.drill.operators.drill import DrillOperator
+from airflow.utils import timezone
+
+DEFAULT_DATE = timezone.datetime(2015, 1, 1)
+DEFAULT_DATE_ISO = DEFAULT_DATE.isoformat()
+DEFAULT_DATE_DS = DEFAULT_DATE_ISO[:10]
+TEST_DAG_ID = 'unit_test_dag'
+
+
+@pytest.mark.backend("drill")
+class TestDrillOperator(unittest.TestCase):
+ def setUp(self):
+ args = {'owner': 'airflow', 'start_date': DEFAULT_DATE}
+ dag = DAG(TEST_DAG_ID, default_args=args)
+ self.dag = dag
+
+ def tearDown(self):
+ tables_to_drop = ['dfs.tmp.test_airflow']
+ from airflow.providers.apache.drill.hooks.drill import DrillHook
+
+ with DrillHook().get_conn() as conn:
+ with conn.cursor() as cur:
+ for table in tables_to_drop:
+ cur.execute(f"DROP TABLE IF EXISTS {table}")
+
+ def test_drill_operator_single(self):
+ sql = """
+ create table dfs.tmp.test_airflow as
+ select * from cp.`employee.json` limit 10
+ """
+ op = DrillOperator(task_id='drill_operator_test_single', sql=sql, dag=self.dag)
+ op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
+
+ def test_drill_operator_multi(self):
+ sql = [
+ "create table dfs.tmp.test_airflow as" "select * from cp.`employee.json` limit 10",
+ "select sum(employee_id), any_value(full_name)" "from dfs.tmp.test_airflow",
+ ]
+ op = DrillOperator(task_id='drill_operator_test_multi', sql=sql, dag=self.dag)
+ op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)