You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by bh...@apache.org on 2022/05/11 18:09:46 UTC
[beam] branch master updated: [BEAM-12526] Add Dependabot (#17563)
This is an automated email from the ASF dual-hosted git repository.
bhulette pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new fbc495c66e7 [BEAM-12526] Add Dependabot (#17563)
fbc495c66e7 is described below
commit fbc495c66e7b36a6033d3ff0c44a5e0050060dec
Author: Danny McCormick <da...@google.com>
AuthorDate: Wed May 11 14:09:39 2022 -0400
[BEAM-12526] Add Dependabot (#17563)
* Create dependabot.yml
* Apache header
* See if dependabot will parse now
* Explanatory comment
* Reduce duplication
* Bug + style from feedback
* Change format to make dependabot happy
---
.github/dependabot.yml | 29 +++++++
sdks/python/setup.py | 230 +++++++++++++++++++++++--------------------------
2 files changed, 139 insertions(+), 120 deletions(-)
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 00000000000..0641e6f267b
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: 2
+updates:
+ - package-ecosystem: "gomod"
+ directory: "/sdks" # Location of package manifests
+ schedule:
+ interval: "daily"
+ - package-ecosystem: "pip"
+ directory: "/sdks/python" # Location of package manifests
+ schedule:
+ interval: "daily"
+ - package-ecosystem: "gradle"
+ directory: "/" # Location of package manifests
+ schedule:
+ interval: "daily"
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index a02f9f9c339..70cec6f00a8 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -120,121 +120,11 @@ try:
except ImportError:
cythonize = lambda *args, **kwargs: []
-REQUIRED_PACKAGES = [
- # Avro 1.9.2 for python3 was broken. The issue was fixed in version 1.9.2.1
- 'crcmod>=1.7,<2.0',
- # dataclasses backport for python_version<3.7. No version bound because this
- # is Python standard since Python 3.7 and each Python version is compatible
- # with a specific dataclasses version.
- 'dataclasses;python_version<"3.7"',
- 'orjson<4.0',
- # Dill doesn't have forwards-compatibility guarantees within minor version.
- # Pickles created with a new version of dill may not unpickle using older
- # version of dill. It is best to use the same version of dill on client and
- # server, therefore list of allowed versions is very narrow.
- # See: https://github.com/uqfoundation/dill/issues/341.
- 'dill>=0.3.1.1,<0.3.2',
- 'cloudpickle>=2.0.0,<3',
- 'fastavro>=0.23.6,<2',
- 'grpcio>=1.29.0,<2',
- 'hdfs>=2.1.0,<3.0.0',
- 'httplib2>=0.8,<0.21.0',
- 'numpy>=1.14.3,<1.23.0',
- 'pymongo>=3.8.0,<4.0.0',
- 'protobuf>=3.12.2,<4',
- 'proto-plus>=1.7.1,<2',
- 'pyarrow>=0.15.1,<8.0.0',
- 'pydot>=1.2.0,<2',
- 'python-dateutil>=2.8.0,<3',
- 'pytz>=2018.3',
- 'requests>=2.24.0,<3.0.0',
- 'typing-extensions>=3.7.0',
-]
-
# [BEAM-8181] pyarrow cannot be installed on 32-bit Windows platforms.
if sys.platform == 'win32' and sys.maxsize <= 2**32:
- REQUIRED_PACKAGES = [
- p for p in REQUIRED_PACKAGES if not p.startswith('pyarrow')
- ]
-
-REQUIRED_TEST_PACKAGES = [
- 'freezegun>=0.3.12',
- 'joblib>=1.0.1',
- 'mock>=1.0.1,<3.0.0',
- 'pandas<2.0.0',
- 'parameterized>=0.7.1,<0.8.0',
- 'pyhamcrest>=1.9,!=1.10.0,<2.0.0',
- 'pyyaml>=3.12,<7.0.0',
- 'requests_mock>=1.7,<2.0',
- 'tenacity>=5.0.2,<6.0',
- 'pytest>=4.4.0,<5.0',
- 'pytest-xdist>=1.29.0,<2',
- 'pytest-timeout>=1.3.3,<2',
- 'scikit-learn>=0.20.0',
- 'sqlalchemy>=1.3,<2.0',
- 'psycopg2-binary>=2.8.5,<3.0.0',
- 'testcontainers[mysql]>=3.0.3,<4.0.0',
- 'cryptography>=36.0.0',
-]
-
-GCP_REQUIREMENTS = [
- 'cachetools>=3.1.0,<5',
- 'google-apitools>=0.5.31,<0.5.32',
- # NOTE: Maintainers, please do not require google-auth>=2.x.x
- # Until this issue is closed
- # https://github.com/googleapis/google-cloud-python/issues/10566
- 'google-auth>=1.18.0,<3',
- 'google-auth-httplib2>=0.1.0,<0.2.0',
- 'google-cloud-datastore>=1.8.0,<2',
- 'google-cloud-pubsub>=2.1.0,<3',
- 'google-cloud-pubsublite>=1.2.0,<2',
- # GCP packages required by tests
- 'google-cloud-bigquery>=1.6.0,<3',
- 'google-cloud-bigquery-storage>=2.6.3',
- 'google-cloud-core>=0.28.1,<2',
- 'google-cloud-bigtable>=0.31.1,<2',
- 'google-cloud-spanner>=1.13.0,<2',
- 'grpcio-gcp>=0.2.2,<1',
- # GCP Packages required by ML functionality
- 'google-cloud-dlp>=3.0.0,<4',
- 'google-cloud-language>=1.3.0,<2',
- 'google-cloud-videointelligence>=1.8.0,<2',
- 'google-cloud-vision>=0.38.0,<2',
- 'google-cloud-recommendations-ai>=0.1.0,<=0.2.0'
-]
-
-INTERACTIVE_BEAM = [
- 'facets-overview>=1.0.0,<2',
- 'google-cloud-dataproc>=3.0.0,<3.2.0',
- # IPython>=8 is not compatible with Python<=3.7
- 'ipython>=7,<8;python_version<="3.7"',
- 'ipython>=8,<9;python_version>"3.7"',
- 'ipykernel>=6,<7',
- 'ipywidgets>=7.6.5,<8',
- # Skip version 6.1.13 due to
- # https://github.com/jupyter/jupyter_client/issues/637
- 'jupyter-client>=6.1.11,<6.1.13',
- 'timeloop>=1.0.2,<2',
-]
-
-INTERACTIVE_BEAM_TEST = [
- # notebok utils
- 'nbformat>=5.0.5,<6',
- 'nbconvert>=6.2.0,<7',
- # headless chrome based integration tests
- 'needle>=0.5.0,<1',
- 'chromedriver-binary>=100,<101',
- # use a fixed major version of PIL for different python versions
- 'pillow>=7.1.1,<8',
-]
-
-AWS_REQUIREMENTS = ['boto3 >=1.9']
-
-AZURE_REQUIREMENTS = [
- 'azure-storage-blob >=12.3.2',
- 'azure-core >=1.7.0',
-]
-
+ pyarrow_dependency = ''
+else:
+ pyarrow_dependency = 'pyarrow>=0.15.1,<8.0.0'
# We must generate protos after setup_requires are installed.
def generate_protos_first():
@@ -272,6 +162,8 @@ if __name__ == '__main__':
# structure must exist before the call to setuptools.find_packages()
# executes below.
generate_protos_first()
+ # Keep all dependencies inlined in the setup call, otherwise Dependabot won't
+ # be able to parse it.
setuptools.setup(
name=PACKAGE_NAME,
version=PACKAGE_VERSION,
@@ -309,7 +201,39 @@ if __name__ == '__main__':
'apache_beam/utils/counters.py',
'apache_beam/utils/windowed_value.py',
]),
- install_requires=REQUIRED_PACKAGES,
+ install_requires=[
+ # Avro 1.9.2 for python3 was broken.
+ # The issue was fixed in version 1.9.2.1
+ 'crcmod>=1.7,<2.0',
+ # dataclasses backport for python_version<3.7. No version bound because
+ # this is Python standard since Python 3.7 and each Python version is
+ # compatible with a specific dataclasses version.
+ 'dataclasses;python_version<"3.7"',
+ 'orjson<4.0',
+ # Dill doesn't have forwards-compatibility guarantees within minor
+ # version. Pickles created with a new version of dill may not unpickle
+ # using older version of dill. It is best to use the same version of
+ # dill on client and server, therefore list of allowed versions is very
+ # narrow. See: https://github.com/uqfoundation/dill/issues/341.
+ 'dill>=0.3.1.1,<0.3.2',
+ 'cloudpickle>=2.0.0,<3',
+ 'fastavro>=0.23.6,<2',
+ 'grpcio>=1.29.0,<2',
+ 'hdfs>=2.1.0,<3.0.0',
+ 'httplib2>=0.8,<0.21.0',
+ 'numpy>=1.14.3,<1.23.0',
+ 'pymongo>=3.8.0,<4.0.0',
+ 'protobuf>=3.12.2,<4',
+ 'proto-plus>=1.7.1,<2',
+ 'pydot>=1.2.0,<2',
+ 'python-dateutil>=2.8.0,<3',
+ 'pytz>=2018.3',
+ 'requests>=2.24.0,<3.0.0',
+ 'typing-extensions>=3.7.0',
+ # Dynamic dependencies must be specified in a separate list, otherwise
+ # Dependabot won't be able to parse the main list. Any dynamic
+ # dependencies will not receive updates from Dependabot.
+ ] + [pyarrow_dependency],
python_requires=python_requires,
# BEAM-8840: Do NOT use tests_require or setup_requires.
extras_require={
@@ -319,12 +243,78 @@ if __name__ == '__main__':
# https://github.com/sphinx-doc/sphinx/issues/9727
'docutils==0.17.1'
],
- 'test': REQUIRED_TEST_PACKAGES,
- 'gcp': GCP_REQUIREMENTS,
- 'interactive': INTERACTIVE_BEAM,
- 'interactive_test': INTERACTIVE_BEAM_TEST,
- 'aws': AWS_REQUIREMENTS,
- 'azure': AZURE_REQUIREMENTS,
+ 'test': [
+ 'freezegun>=0.3.12',
+ 'joblib>=1.0.1',
+ 'mock>=1.0.1,<3.0.0',
+ 'pandas<2.0.0',
+ 'parameterized>=0.7.1,<0.8.0',
+ 'pyhamcrest>=1.9,!=1.10.0,<2.0.0',
+ 'pyyaml>=3.12,<7.0.0',
+ 'requests_mock>=1.7,<2.0',
+ 'tenacity>=5.0.2,<6.0',
+ 'pytest>=4.4.0,<5.0',
+ 'pytest-xdist>=1.29.0,<2',
+ 'pytest-timeout>=1.3.3,<2',
+ 'scikit-learn>=0.20.0',
+ 'sqlalchemy>=1.3,<2.0',
+ 'psycopg2-binary>=2.8.5,<3.0.0',
+ 'testcontainers[mysql]>=3.0.3,<4.0.0',
+ 'cryptography>=36.0.0',
+ ],
+ 'gcp': [
+ 'cachetools>=3.1.0,<5',
+ 'google-apitools>=0.5.31,<0.5.32',
+ # NOTE: Maintainers, please do not require google-auth>=2.x.x
+ # Until this issue is closed
+ # https://github.com/googleapis/google-cloud-python/issues/10566
+ 'google-auth>=1.18.0,<3',
+ 'google-auth-httplib2>=0.1.0,<0.2.0',
+ 'google-cloud-datastore>=1.8.0,<2',
+ 'google-cloud-pubsub>=2.1.0,<3',
+ 'google-cloud-pubsublite>=1.2.0,<2',
+ # GCP packages required by tests
+ 'google-cloud-bigquery>=1.6.0,<3',
+ 'google-cloud-bigquery-storage>=2.6.3',
+ 'google-cloud-core>=0.28.1,<2',
+ 'google-cloud-bigtable>=0.31.1,<2',
+ 'google-cloud-spanner>=1.13.0,<2',
+ 'grpcio-gcp>=0.2.2,<1',
+ # GCP Packages required by ML functionality
+ 'google-cloud-dlp>=3.0.0,<4',
+ 'google-cloud-language>=1.3.0,<2',
+ 'google-cloud-videointelligence>=1.8.0,<2',
+ 'google-cloud-vision>=0.38.0,<2',
+ 'google-cloud-recommendations-ai>=0.1.0,<=0.2.0'
+ ],
+ 'interactive': [
+ 'facets-overview>=1.0.0,<2',
+ 'google-cloud-dataproc>=3.0.0,<3.2.0',
+ # IPython>=8 is not compatible with Python<=3.7
+ 'ipython>=7,<8;python_version<="3.7"',
+ 'ipython>=8,<9;python_version>"3.7"',
+ 'ipykernel>=6,<7',
+ 'ipywidgets>=7.6.5,<8',
+ # Skip version 6.1.13 due to
+ # https://github.com/jupyter/jupyter_client/issues/637
+ 'jupyter-client>=6.1.11,<6.1.13',
+ 'timeloop>=1.0.2,<2',
+ ],
+ 'interactive_test': [
+ # notebok utils
+ 'nbformat>=5.0.5,<6',
+ 'nbconvert>=6.2.0,<7',
+ # headless chrome based integration tests
+ 'needle>=0.5.0,<1',
+ 'chromedriver-binary>=100,<101',
+ # use a fixed major version of PIL for different python versions
+ 'pillow>=7.1.1,<8',
+ ],
+ 'aws': ['boto3 >=1.9'],
+ 'azure': [
+ 'azure-storage-blob >=12.3.2',
+ 'azure-core >=1.7.0',
+ ],
'dataframe': ['pandas>=1.0,<1.5']
},
zip_safe=False,