You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by tv...@apache.org on 2022/07/27 22:26:13 UTC
[beam] branch master updated: Replace distutils with supported modules. (#22456)
This is an automated email from the ASF dual-hosted git repository.
tvalentyn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 8652c452419 Replace distutils with supported modules. (#22456)
8652c452419 is described below
commit 8652c4524194a8c0cc3cf7abd5d24aa4f4c566c1
Author: Anand Inguva <34...@users.noreply.github.com>
AuthorDate: Thu Jul 28 03:56:04 2022 +0530
Replace distutils with supported modules. (#22456)
Co-authored-by: Anand Inguva <an...@google.com>
---
.github/workflows/build_wheels.yml | 2 +-
.../beam/sdk/extensions/python/bootstrap_beam_venv.py | 6 +++---
sdks/python/apache_beam/examples/complete/juliaset/setup.py | 2 +-
sdks/python/apache_beam/runners/portability/stager.py | 4 ++--
sdks/python/container/Dockerfile | 2 ++
sdks/python/setup.py | 13 +++++++++----
.../en/documentation/sdks/python-pipeline-dependencies.md | 2 +-
7 files changed, 19 insertions(+), 12 deletions(-)
diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
index f2066b14e18..520b97bcd9f 100644
--- a/.github/workflows/build_wheels.yml
+++ b/.github/workflows/build_wheels.yml
@@ -240,7 +240,7 @@ jobs:
working-directory: apache-beam-source
env:
CIBW_BUILD: ${{ matrix.os_python.python }}
- CIBW_BEFORE_BUILD: pip install cython
+ CIBW_BEFORE_BUILD: pip install cython && pip install --upgrade setuptools
run: cibuildwheel --print-build-identifiers && cibuildwheel --output-dir wheelhouse
shell: bash
- name: install sha512sum on MacOS
diff --git a/sdks/java/extensions/python/src/main/resources/org/apache/beam/sdk/extensions/python/bootstrap_beam_venv.py b/sdks/java/extensions/python/src/main/resources/org/apache/beam/sdk/extensions/python/bootstrap_beam_venv.py
index a8f6b33d9c3..54aeb6a1ee3 100644
--- a/sdks/java/extensions/python/src/main/resources/org/apache/beam/sdk/extensions/python/bootstrap_beam_venv.py
+++ b/sdks/java/extensions/python/src/main/resources/org/apache/beam/sdk/extensions/python/bootstrap_beam_venv.py
@@ -24,13 +24,13 @@ suitable python executable.
"""
import argparse
-import distutils.version
import hashlib
import json
import os
import shutil
import subprocess
import sys
+from pkg_resources import parse_version
def main():
@@ -66,9 +66,9 @@ def main():
def maybe_strict_version(s):
try:
- return distutils.version.StrictVersion(s)
+ return parse_version(s)
except:
- return distutils.version.StrictVersion('0.0')
+ return parse_version('0.0')
beam_version = max(info['releases'], key=maybe_strict_version)
beam_package = 'apache_beam[gcp,aws,asure,dataframe]==' + beam_version
diff --git a/sdks/python/apache_beam/examples/complete/juliaset/setup.py b/sdks/python/apache_beam/examples/complete/juliaset/setup.py
index c4dcbe12113..b65eb15a3f2 100644
--- a/sdks/python/apache_beam/examples/complete/juliaset/setup.py
+++ b/sdks/python/apache_beam/examples/complete/juliaset/setup.py
@@ -28,9 +28,9 @@ when running the workflow for remote execution.
# pytype: skip-file
import subprocess
-from distutils.command.build import build as _build # type: ignore
import setuptools
+from setuptools.command.build import build as _build # type: ignore
# This class handles the pip install mechanism.
diff --git a/sdks/python/apache_beam/runners/portability/stager.py b/sdks/python/apache_beam/runners/portability/stager.py
index 743beb490ed..e06c71c917d 100644
--- a/sdks/python/apache_beam/runners/portability/stager.py
+++ b/sdks/python/apache_beam/runners/portability/stager.py
@@ -54,7 +54,6 @@ import os
import shutil
import sys
import tempfile
-from distutils.version import StrictVersion
from typing import Callable
from typing import List
from typing import Optional
@@ -62,6 +61,7 @@ from typing import Tuple
from urllib.parse import urlparse
import pkg_resources
+from pkg_resources import parse_version
from apache_beam.internal import pickler
from apache_beam.internal.http_client import get_new_http
@@ -698,7 +698,7 @@ class Stager(object):
# addressed, download wheel based on glibc version in Beam's Python
# Base image
pip_version = pkg_resources.get_distribution('pip').version
- if StrictVersion(pip_version) >= StrictVersion('19.3'):
+ if parse_version(pip_version) >= parse_version('19.3'):
return 'manylinux2014_x86_64'
else:
return 'manylinux2010_x86_64'
diff --git a/sdks/python/container/Dockerfile b/sdks/python/container/Dockerfile
index b40eac647b4..a301db74ee0 100644
--- a/sdks/python/container/Dockerfile
+++ b/sdks/python/container/Dockerfile
@@ -48,6 +48,8 @@ RUN \
rm -rf /root/.cache/pip && \
rm -rf /tmp/base_image_requirements.txt
+RUN pip install --upgrade pip setuptools
+
# Install Google Cloud SDK.
ENV CLOUDSDK_CORE_DISABLE_PROMPTS yes
ENV PATH $PATH:/usr/local/gcloud/google-cloud-sdk/bin
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index ef8f73a0477..1753ec9d6a5 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -20,8 +20,6 @@
import os
import sys
import warnings
-from distutils.errors import DistutilsError
-from distutils.version import StrictVersion
from pathlib import Path
# Pylint and isort disagree here.
@@ -30,9 +28,16 @@ import setuptools
from pkg_resources import DistributionNotFound
from pkg_resources import get_distribution
from pkg_resources import normalize_path
+from pkg_resources import parse_version
from pkg_resources import to_filename
from setuptools import Command
+# pylint: disable=wrong-import-order
+# It is recommended to import setuptools prior to importing distutils to avoid
+# using legacy behavior from distutils.
+# https://setuptools.readthedocs.io/en/latest/history.html#v48-0-0
+from distutils.errors import DistutilsError # isort:skip
+
class mypy(Command):
user_options = []
@@ -92,7 +97,7 @@ different technologies and user communities.
REQUIRED_PIP_VERSION = '7.0.0'
_PIP_VERSION = get_distribution('pip').version
-if StrictVersion(_PIP_VERSION) < StrictVersion(REQUIRED_PIP_VERSION):
+if parse_version(_PIP_VERSION) < parse_version(REQUIRED_PIP_VERSION):
warnings.warn(
"You are using version {0} of pip. " \
"However, version {1} is recommended.".format(
@@ -103,7 +108,7 @@ if StrictVersion(_PIP_VERSION) < StrictVersion(REQUIRED_PIP_VERSION):
REQUIRED_CYTHON_VERSION = '0.28.1'
try:
_CYTHON_VERSION = get_distribution('cython').version
- if StrictVersion(_CYTHON_VERSION) < StrictVersion(REQUIRED_CYTHON_VERSION):
+ if parse_version(_CYTHON_VERSION) < parse_version(REQUIRED_CYTHON_VERSION):
warnings.warn(
"You are using version {0} of cython. " \
"However, version {1} is recommended.".format(
diff --git a/website/www/site/content/en/documentation/sdks/python-pipeline-dependencies.md b/website/www/site/content/en/documentation/sdks/python-pipeline-dependencies.md
index ca91194c533..bf2e44e5586 100644
--- a/website/www/site/content/en/documentation/sdks/python-pipeline-dependencies.md
+++ b/website/www/site/content/en/documentation/sdks/python-pipeline-dependencies.md
@@ -77,7 +77,7 @@ If your pipeline uses packages that are not available publicly (e.g. packages th
python setup.py sdist
- See the [sdist documentation](https://docs.python.org/2/distutils/sourcedist.html) for more details on this command.
+ See the [sdist documentation](https://docs.python.org/3/distutils/sourcedist.html) for more details on this command.
## Multiple File Dependencies