You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by tv...@apache.org on 2022/11/11 20:57:41 UTC
[beam] branch master updated: Update staging of Python wheels (#24114)
This is an automated email from the ASF dual-hosted git repository.
tvalentyn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 5d2dbf957e4 Update staging of Python wheels (#24114)
5d2dbf957e4 is described below
commit 5d2dbf957e4e82fb3980726940df02ac67e563cd
Author: Anand Inguva <34...@users.noreply.github.com>
AuthorDate: Fri Nov 11 15:57:28 2022 -0500
Update staging of Python wheels (#24114)
Fixes https://github.com/apache/beam/issues/24110
---
.../python/apache_beam/runners/portability/stager.py | 20 ++++++++++++++------
.../apache_beam/runners/portability/stager_test.py | 2 ++
sdks/python/container/boot.go | 6 +++---
sdks/python/setup.py | 2 +-
4 files changed, 20 insertions(+), 10 deletions(-)
diff --git a/sdks/python/apache_beam/runners/portability/stager.py b/sdks/python/apache_beam/runners/portability/stager.py
index abcef4679c2..6ebdd3f2854 100644
--- a/sdks/python/apache_beam/runners/portability/stager.py
+++ b/sdks/python/apache_beam/runners/portability/stager.py
@@ -723,6 +723,8 @@ class Stager(object):
# Base image
pip_version = pkg_resources.get_distribution('pip').version
if parse_version(pip_version) >= parse_version('19.3'):
+ # pip can only recognize manylinux2014_x86_64 wheels
+ # from version 19.3.
return 'manylinux2014_x86_64'
else:
return 'manylinux2010_x86_64'
@@ -854,13 +856,15 @@ class Stager(object):
try:
abi_suffix = 'm' if sys.version_info < (3, 8) else ''
# Stage binary distribution of the SDK, for now on a best-effort basis.
+ platform_tag = Stager._get_platform_for_default_sdk_container()
sdk_local_file = Stager._download_pypi_sdk_package(
temp_dir,
fetch_binary=True,
language_version_tag='%d%d' %
(sys.version_info[0], sys.version_info[1]),
abi_tag='cp%d%d%s' %
- (sys.version_info[0], sys.version_info[1], abi_suffix))
+ (sys.version_info[0], sys.version_info[1], abi_suffix),
+ platform_tag=platform_tag)
sdk_binary_staged_name = Stager.\
_desired_sdk_filename_in_staging_location(sdk_local_file)
_LOGGER.info(
@@ -897,10 +901,10 @@ class Stager(object):
def _download_pypi_sdk_package(
temp_dir,
fetch_binary=False,
- language_version_tag='27',
+ language_version_tag='39',
language_implementation_tag='cp',
- abi_tag='cp27mu',
- platform_tag='manylinux1_x86_64'):
+ abi_tag='cp39',
+ platform_tag='manylinux2014_x86_64'):
"""Downloads SDK package from PyPI and returns path to local path."""
package_name = Stager.get_sdk_package_name()
try:
@@ -935,7 +939,10 @@ class Stager(object):
'--platform',
platform_tag
])
- # Example wheel: apache_beam-2.4.0-cp27-cp27mu-manylinux1_x86_64.whl
+ # Example wheel: with manylinux14 tag.
+ # apache_beam-2.43.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl # pylint: disable=line-too-long
+ if platform_tag == 'manylinux2014_x86_64':
+ platform_tag = 'manylinux_2_17_x86_64.' + platform_tag
expected_files = [
os.path.join(
temp_dir,
@@ -945,8 +952,9 @@ class Stager(object):
language_implementation_tag,
language_version_tag,
abi_tag,
- platform_tag))
+ platform_tag)),
]
+
else:
_LOGGER.info('Downloading source distribution of the SDK from PyPi')
cmd_args.extend(['--no-binary', ':all:'])
diff --git a/sdks/python/apache_beam/runners/portability/stager_test.py b/sdks/python/apache_beam/runners/portability/stager_test.py
index c1806c38494..bf876925ff7 100644
--- a/sdks/python/apache_beam/runners/portability/stager_test.py
+++ b/sdks/python/apache_beam/runners/portability/stager_test.py
@@ -137,6 +137,8 @@ class StagerTest(unittest.TestCase):
# Per PEP-0427 in wheel filenames non-alphanumeric characters
# in distribution name are replaced with underscore.
distribution_name = distribution_name.replace('-', '_')
+ if args[17] == 'manylinux2014_x86_64':
+ args[17] = 'manylinux_2_17_x86_64.' + args[17]
package_file = '%s-%s-%s%s-%s-%s.whl' % (
distribution_name,
distribution_version,
diff --git a/sdks/python/container/boot.go b/sdks/python/container/boot.go
index 646109127f1..a9ae893ab0c 100644
--- a/sdks/python/container/boot.go
+++ b/sdks/python/container/boot.go
@@ -324,7 +324,7 @@ func setupAcceptableWheelSpecs() error {
if err != nil {
return err
}
- re := regexp.MustCompile(`Python (\d)\.(\d).*`)
+ re := regexp.MustCompile(`Python (\d)\.(\d+).*`)
pyVersions := re.FindStringSubmatch(string(stdoutStderr[:]))
if len(pyVersions) != 3 {
return fmt.Errorf("cannot get parse Python version from %s", stdoutStderr)
@@ -333,9 +333,9 @@ func setupAcceptableWheelSpecs() error {
var wheelName string
switch pyVersion {
case "36", "37":
- wheelName = fmt.Sprintf("cp%s-cp%sm-manylinux1_x86_64.whl", pyVersion, pyVersion)
+ wheelName = fmt.Sprintf("cp%s-cp%sm-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", pyVersion, pyVersion)
default:
- wheelName = fmt.Sprintf("cp%s-cp%s-manylinux1_x86_64.whl", pyVersion, pyVersion)
+ wheelName = fmt.Sprintf("cp%s-cp%s-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", pyVersion, pyVersion)
}
acceptableWhlSpecs = append(acceptableWhlSpecs, wheelName)
return nil
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index 61858fa5d97..dddceed4e08 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -95,7 +95,7 @@ execution engines and providing extensibility points for connecting to
different technologies and user communities.
'''
-RECOMMENDED_MIN_PIP_VERSION = '7.0.0'
+RECOMMENDED_MIN_PIP_VERSION = '19.3.0'
try:
_PIP_VERSION = get_distribution('pip').version
if parse_version(_PIP_VERSION) < parse_version(RECOMMENDED_MIN_PIP_VERSION):