You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2021/06/22 19:24:42 UTC
[airflow] 04/47: Move plyvel to google provider extra (#15812)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-1-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 03b1222d5cb36dab7d41098f3fd76905626573bd
Author: Daniel Standish <15...@users.noreply.github.com>
AuthorDate: Tue May 18 22:59:58 2021 -0700
Move plyvel to google provider extra (#15812)
Plyvel does not build on macOS without levelDB installed in system. Its better to make it an optional install.
(cherry picked from commit 76a80bb17c9cecbe3767dab471f6a79084c822ea)
---
CONTRIBUTING.rst | 10 +++++-----
INSTALL | 10 +++++-----
airflow/providers/google/provider.yaml | 1 +
.../operators/leveldb/leveldb.rst | 7 ++++++-
docs/apache-airflow/extra-packages-ref.rst | 2 ++
docs/spelling_wordlist.txt | 1 +
setup.py | 3 ++-
7 files changed, 22 insertions(+), 12 deletions(-)
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index 8328f57..94f325f 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -583,11 +583,11 @@ apache.sqoop, apache.webhdfs, asana, async, atlas, aws, azure, cassandra, celery
cncf.kubernetes, crypto, dask, databricks, datadog, deprecated_api, devel, devel_all, devel_ci,
devel_hadoop, dingding, discord, doc, docker, druid, elasticsearch, exasol, facebook, ftp, gcp,
gcp_api, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, imap, jdbc,
-jenkins, jira, kerberos, kubernetes, ldap, microsoft.azure, microsoft.mssql, microsoft.winrm, mongo,
-mssql, mysql, neo4j, odbc, openfaas, opsgenie, oracle, pagerduty, papermill, password, pinot,
-plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid,
-sentry, sftp, singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau, telegram, trino,
-vertica, virtualenv, webhdfs, winrm, yandex, zendesk
+jenkins, jira, kerberos, kubernetes, ldap, leveldb, microsoft.azure, microsoft.mssql,
+microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openfaas, opsgenie, oracle, pagerduty, papermill,
+password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba,
+segment, sendgrid, sentry, sftp, singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau,
+telegram, trino, vertica, virtualenv, webhdfs, winrm, yandex, zendesk
.. END EXTRAS HERE
diff --git a/INSTALL b/INSTALL
index c488f73..50ac150 100644
--- a/INSTALL
+++ b/INSTALL
@@ -95,11 +95,11 @@ apache.sqoop, apache.webhdfs, asana, async, atlas, aws, azure, cassandra, celery
cncf.kubernetes, crypto, dask, databricks, datadog, deprecated_api, devel, devel_all, devel_ci,
devel_hadoop, dingding, discord, doc, docker, druid, elasticsearch, exasol, facebook, ftp, gcp,
gcp_api, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, imap, jdbc,
-jenkins, jira, kerberos, kubernetes, ldap, microsoft.azure, microsoft.mssql, microsoft.winrm, mongo,
-mssql, mysql, neo4j, odbc, openfaas, opsgenie, oracle, pagerduty, papermill, password, pinot,
-plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid,
-sentry, sftp, singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau, telegram, trino,
-vertica, virtualenv, webhdfs, winrm, yandex, zendesk
+jenkins, jira, kerberos, kubernetes, ldap, leveldb, microsoft.azure, microsoft.mssql,
+microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openfaas, opsgenie, oracle, pagerduty, papermill,
+password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba,
+segment, sendgrid, sentry, sftp, singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau,
+telegram, trino, vertica, virtualenv, webhdfs, winrm, yandex, zendesk
# END EXTRAS HERE
diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml
index 87ba4ae..9af2070 100644
--- a/airflow/providers/google/provider.yaml
+++ b/airflow/providers/google/provider.yaml
@@ -748,3 +748,4 @@ extra-links:
additional-extras:
apache.beam: apache-beam[gcp]
+ leveldb: plyvel
diff --git a/docs/apache-airflow-providers-google/operators/leveldb/leveldb.rst b/docs/apache-airflow-providers-google/operators/leveldb/leveldb.rst
index b31bf86..2cb8ff4 100644
--- a/docs/apache-airflow-providers-google/operators/leveldb/leveldb.rst
+++ b/docs/apache-airflow-providers-google/operators/leveldb/leveldb.rst
@@ -17,6 +17,8 @@
+.. _howto/operator:LevelDBOperator:
+
Google LevelDB Operator
================================
@@ -27,7 +29,10 @@ an ordered mapping from string keys to string values.
:depth: 1
:local:
-.. _howto/operator:LevelDBOperator:
+.. note::
+
+ To use LevelDB hooks and operators you must requires installation of ``plyvel``. It will be
+ installed if you specify the extra ``apache-airflow-providers-google[leveldb]``.
Put key
^^^^^^^^^^^^^^^
diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst
index 7d218ab..b4b4bb4 100644
--- a/docs/apache-airflow/extra-packages-ref.rst
+++ b/docs/apache-airflow/extra-packages-ref.rst
@@ -60,6 +60,8 @@ python dependencies for the provided package.
+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+
| ldap | ``pip install 'apache-airflow[ldap]'`` | LDAP authentication for users |
+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+
+| leveldb | ``pip install 'apache-airflow[leveldb]'`` | Required for use leveldb extra in google provider |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+
| password | ``pip install 'apache-airflow[password]'`` | Password authentication for users |
+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+
| rabbitmq | ``pip install 'apache-airflow[rabbitmq]'`` | RabbitMQ support as a Celery backend |
diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt
index 3c38bbf..f1f8ab5 100644
--- a/docs/spelling_wordlist.txt
+++ b/docs/spelling_wordlist.txt
@@ -1430,6 +1430,7 @@ utils
uuid
validator
vals
+ve
vendored
venvs
versionable
diff --git a/setup.py b/setup.py
index d8d170d..85237a1 100644
--- a/setup.py
+++ b/setup.py
@@ -326,7 +326,6 @@ google = [
# pandas-gbq 0.15.0 release broke google provider's bigquery import
# _check_google_client_version (airflow/providers/google/cloud/hooks/bigquery.py:49)
'pandas-gbq<0.15.0',
- 'plyvel',
]
grpc = [
'google-auth>=1.0.0, <2.0.0dev',
@@ -381,6 +380,7 @@ ldap = [
'ldap3>=2.5.1',
'python-ldap',
]
+leveldb = ['plyvel']
mongo = [
'dnspython>=1.13.0,<2.0.0',
'pymongo>=3.6.0',
@@ -636,6 +636,7 @@ CORE_EXTRAS_REQUIREMENTS: Dict[str, List[str]] = {
'google_auth': flask_oauth,
'kerberos': kerberos,
'ldap': ldap,
+ 'leveldb': leveldb,
'password': password,
'rabbitmq': rabbitmq,
'sentry': sentry,