You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by da...@apache.org on 2023/08/22 14:51:10 UTC
[beam] branch revert-27436-unstage_sdk created (now ace89c994b7)
This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a change to branch revert-27436-unstage_sdk
in repository https://gitbox.apache.org/repos/asf/beam.git
at ace89c994b7 Revert "[Python]Don't stage beam SDK in Stager (#27436)"
This branch includes the following new commits:
new ace89c994b7 Revert "[Python]Don't stage beam SDK in Stager (#27436)"
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[beam] 01/01: Revert "[Python]Don't stage beam SDK in Stager (#27436)"
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a commit to branch revert-27436-unstage_sdk
in repository https://gitbox.apache.org/repos/asf/beam.git
commit ace89c994b76308f6201a5e2a8842988a76ad6d8
Author: Danny McCormick <da...@google.com>
AuthorDate: Tue Aug 22 10:51:01 2023 -0400
Revert "[Python]Don't stage beam SDK in Stager (#27436)"
This reverts commit 408d766d55fb56eb2af2b65d72a27618cda84883.
---
CHANGES.md | 1 -
.../python/apache_beam/options/pipeline_options.py | 10 +-
.../apache_beam/runners/portability/stager.py | 192 +++++++++++++++++----
.../apache_beam/runners/portability/stager_test.py | 92 +++++++++-
sdks/python/container/boot.go | 5 -
sdks/python/container/piputil.go | 13 --
6 files changed, 255 insertions(+), 58 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index 29b8fb2e0ab..cff41bc5971 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -126,7 +126,6 @@
## Breaking Changes
* Python SDK: Legacy runner support removed from Dataflow, all pipelines must use runner v2.
-* [Python] Dataflow Runner will no longer stage Beam SDK from PyPI in the `--staging_location` at pipeline submission. Custom container images that are not based on Beam's default image must include Apache Beam installation.([#26996](https://github.com/apache/beam/issues/26996))
## Deprecations
diff --git a/sdks/python/apache_beam/options/pipeline_options.py b/sdks/python/apache_beam/options/pipeline_options.py
index 6d9c5ecd37b..5d05259ea71 100644
--- a/sdks/python/apache_beam/options/pipeline_options.py
+++ b/sdks/python/apache_beam/options/pipeline_options.py
@@ -1296,13 +1296,11 @@ class SetupOptions(PipelineOptions):
'--sdk_location',
default='default',
help=(
- 'Path to a custom Beam SDK package to install and use on the'
- 'runner. It can be a URL, a GCS path, or a local path to an '
+ 'Override the default location from where the Beam SDK is '
+ 'downloaded. It can be a URL, a GCS path, or a local path to an '
'SDK tarball. Workflow submissions will download or copy an SDK '
- 'tarball from here. If set to "default", '
- 'runners will use the SDK provided in the default environment.'
- 'Use this flag when running pipelines with an unreleased or '
- 'manually patched version of Beam SDK.'))
+ 'tarball from here. If set to the string "default", a standard '
+ 'SDK location is used. If empty, no SDK is copied.'))
parser.add_argument(
'--extra_package',
'--extra_packages',
diff --git a/sdks/python/apache_beam/runners/portability/stager.py b/sdks/python/apache_beam/runners/portability/stager.py
index 1f093b1d7bc..ee920a32516 100644
--- a/sdks/python/apache_beam/runners/portability/stager.py
+++ b/sdks/python/apache_beam/runners/portability/stager.py
@@ -115,6 +115,12 @@ class Stager(object):
"""Commits manifest."""
raise NotImplementedError
+ @staticmethod
+ def get_sdk_package_name():
+ """For internal use only; no backwards-compatibility guarantees.
+ Returns the PyPI package name to be staged."""
+ return names.BEAM_PACKAGE_NAME
+
@staticmethod
def _create_file_stage_to_artifact(local_path, staged_name):
return beam_runner_api_pb2.ArtifactInformation(
@@ -290,28 +296,30 @@ class Stager(object):
setup_options.extra_packages, temp_dir=temp_dir))
if hasattr(setup_options, 'sdk_location'):
- sdk_location = setup_options.sdk_location
- if Stager._is_remote_path(sdk_location):
- try:
- resources.extend(
- Stager._create_beam_sdk(
- sdk_remote_location=setup_options.sdk_location,
- temp_dir=temp_dir,
- ))
- except:
- raise RuntimeError(
- 'The --sdk_location option was used with an unsupported '
- 'type of location: %s' % sdk_location)
-
- elif sdk_location == 'default':
- # Use default location for a runner.
- pass
- elif sdk_location == 'container':
- # Used in the past to indicate that SDK should be used from container
- # image instead of being staged.
- # Equivalent to 'default' now, leaving for backwards compatibility.
+
+ if (setup_options.sdk_location == 'default') or Stager._is_remote_path(
+ setup_options.sdk_location):
+ # If --sdk_location is not specified then the appropriate package
+ # will be obtained from PyPI (https://pypi.python.org) based on the
+ # version of the currently running SDK. If the option is
+ # present then no version matching is made and the exact URL or path
+ # is expected.
+ #
+ # Unit tests running in the 'python setup.py test' context will
+ # not have the sdk_location attribute present and therefore we
+ # will not stage SDK.
+ sdk_remote_location = 'pypi' if (
+ setup_options.sdk_location == 'default'
+ ) else setup_options.sdk_location
+ resources.extend(
+ Stager._create_beam_sdk(sdk_remote_location, temp_dir))
+ elif setup_options.sdk_location == 'container':
+ # Use the SDK that's built into the container, rather than re-staging
+ # it.
pass
else:
+ # This branch is also used by internal tests running with the SDK
+ # built at head.
if os.path.isdir(setup_options.sdk_location):
sdk_path = os.path.join(
setup_options.sdk_location, names.STAGED_SDK_SOURCES_FILENAME)
@@ -337,6 +345,7 @@ class Stager(object):
raise RuntimeError(
'The file "%s" cannot be found. Its location was specified '
'by the --sdk_location command-line option.' % sdk_path)
+
# The following artifacts are not processed by python sdk container boot
# sequence in a setup mode and hence should not be skipped even if a
# prebuilt sdk container image is used.
@@ -815,7 +824,8 @@ class Stager(object):
Args:
sdk_remote_location: A URL from which the file can be downloaded or a
- remote file location. The SDK file can be a tarball or a wheel.
+ remote file location. The SDK file can be a tarball or a wheel. Set
+ to 'pypi' to download and stage a wheel and source SDK from PyPi.
temp_dir: path to temporary location where the file should be
downloaded.
@@ -826,14 +836,136 @@ class Stager(object):
Raises:
RuntimeError: if staging was not successful.
"""
+ if sdk_remote_location == 'pypi':
+ sdk_local_file = Stager._download_pypi_sdk_package(temp_dir)
+ sdk_sources_staged_name = Stager.\
+ _desired_sdk_filename_in_staging_location(sdk_local_file)
+ _LOGGER.info('Staging SDK sources from PyPI: %s', sdk_sources_staged_name)
+ staged_sdk_files = [
+ Stager._create_file_stage_to_artifact(
+ sdk_local_file, sdk_sources_staged_name)
+ ]
+ try:
+ abi_suffix = 'm' if sys.version_info < (3, 8) else ''
+ # Stage binary distribution of the SDK, for now on a best-effort basis.
+ platform_tag = Stager._get_platform_for_default_sdk_container()
+ sdk_local_file = Stager._download_pypi_sdk_package(
+ temp_dir,
+ fetch_binary=True,
+ language_version_tag='%d%d' %
+ (sys.version_info[0], sys.version_info[1]),
+ abi_tag='cp%d%d%s' %
+ (sys.version_info[0], sys.version_info[1], abi_suffix),
+ platform_tag=platform_tag)
+ sdk_binary_staged_name = Stager.\
+ _desired_sdk_filename_in_staging_location(sdk_local_file)
+ _LOGGER.info(
+ 'Staging binary distribution of the SDK from PyPI: %s',
+ sdk_binary_staged_name)
+ staged_sdk_files.append(
+ Stager._create_file_stage_to_artifact(
+ sdk_local_file, sdk_binary_staged_name))
+ except RuntimeError as e:
+ _LOGGER.warning(
+ 'Failed to download requested binary distribution '
+ 'of the SDK: %s',
+ repr(e))
- sdk_remote_parsed = urlparse(sdk_remote_location)
- sdk_remote_filename = os.path.basename(sdk_remote_parsed.path)
- local_download_file = os.path.join(temp_dir, sdk_remote_filename)
- Stager._download_file(sdk_remote_location, local_download_file)
- staged_name = Stager._desired_sdk_filename_in_staging_location(
- local_download_file)
- _LOGGER.info('Staging Beam SDK from %s', sdk_remote_location)
- return [
- Stager._create_file_stage_to_artifact(local_download_file, staged_name)
+ return staged_sdk_files
+ elif Stager._is_remote_path(sdk_remote_location):
+ sdk_remote_parsed = urlparse(sdk_remote_location)
+ sdk_remote_filename = os.path.basename(sdk_remote_parsed.path)
+ local_download_file = os.path.join(temp_dir, sdk_remote_filename)
+ Stager._download_file(sdk_remote_location, local_download_file)
+ staged_name = Stager._desired_sdk_filename_in_staging_location(
+ local_download_file)
+ _LOGGER.info('Staging Beam SDK from %s', sdk_remote_location)
+ return [
+ Stager._create_file_stage_to_artifact(
+ local_download_file, staged_name)
+ ]
+ else:
+ raise RuntimeError(
+ 'The --sdk_location option was used with an unsupported '
+ 'type of location: %s' % sdk_remote_location)
+
+ @staticmethod
+ def _download_pypi_sdk_package(
+ temp_dir,
+ fetch_binary=False,
+ language_version_tag='39',
+ language_implementation_tag='cp',
+ abi_tag='cp39',
+ platform_tag='manylinux2014_x86_64'):
+ """Downloads SDK package from PyPI and returns path to local path."""
+ package_name = Stager.get_sdk_package_name()
+ try:
+ version = pkg_resources.get_distribution(package_name).version
+ except pkg_resources.DistributionNotFound:
+ raise RuntimeError(
+ 'Please set --sdk_location command-line option '
+ 'or install a valid {} distribution.'.format(package_name))
+ cmd_args = [
+ Stager._get_python_executable(),
+ '-m',
+ 'pip',
+ 'download',
+ '--dest',
+ temp_dir,
+ '%s==%s' % (package_name, version),
+ '--no-deps'
]
+
+ if fetch_binary:
+ _LOGGER.info('Downloading binary distribution of the SDK from PyPi')
+ # Get a wheel distribution for the SDK from PyPI.
+ cmd_args.extend([
+ '--only-binary',
+ ':all:',
+ '--python-version',
+ language_version_tag,
+ '--implementation',
+ language_implementation_tag,
+ '--abi',
+ abi_tag,
+ '--platform',
+ platform_tag
+ ])
+ # Example wheel: with manylinux14 tag.
+ # apache_beam-2.43.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl # pylint: disable=line-too-long
+ if platform_tag == 'manylinux2014_x86_64':
+ platform_tag = 'manylinux_2_17_x86_64.' + platform_tag
+ expected_files = [
+ os.path.join(
+ temp_dir,
+ '%s-%s-%s%s-%s-%s.whl' % (
+ package_name.replace('-', '_'),
+ version,
+ language_implementation_tag,
+ language_version_tag,
+ abi_tag,
+ platform_tag)),
+ ]
+
+ else:
+ _LOGGER.info('Downloading source distribution of the SDK from PyPi')
+ cmd_args.extend(['--no-binary', ':all:'])
+ expected_files = [
+ os.path.join(temp_dir, '%s-%s.zip' % (package_name, version)),
+ os.path.join(temp_dir, '%s-%s.tar.gz' % (package_name, version))
+ ]
+
+ _LOGGER.info('Executing command: %s', cmd_args)
+ try:
+ processes.check_output(cmd_args)
+ except processes.CalledProcessError as e:
+ raise RuntimeError(repr(e))
+
+ for sdk_file in expected_files:
+ if os.path.exists(sdk_file):
+ return sdk_file
+
+ raise RuntimeError(
+ 'Failed to download a distribution for the running SDK. '
+ 'Expected either one of %s to be found in the download folder.' %
+ (expected_files))
diff --git a/sdks/python/apache_beam/runners/portability/stager_test.py b/sdks/python/apache_beam/runners/portability/stager_test.py
index 839f7e57773..bf876925ff7 100644
--- a/sdks/python/apache_beam/runners/portability/stager_test.py
+++ b/sdks/python/apache_beam/runners/portability/stager_test.py
@@ -96,6 +96,64 @@ class StagerTest(unittest.TestCase):
self.create_temp_file(os.path.join(cache_dir, 'abc.txt'), 'nothing')
self.create_temp_file(os.path.join(cache_dir, 'def.txt'), 'nothing')
+ def build_fake_pip_download_command_handler(self, has_wheels):
+ """A stub for apache_beam.utils.processes.check_output that imitates pip.
+
+ Args:
+ has_wheels: Whether pip fake should have a whl distribution of packages.
+ """
+ def pip_fake(args):
+ """Fakes fetching a package from pip by creating a temporary file.
+
+ Args:
+ args: a complete list of command line arguments to invoke pip.
+ The fake is sensitive to the order of the arguments.
+ Supported commands:
+
+ 1) Download SDK sources file:
+ python pip -m download --dest /tmp/dir apache-beam==2.0.0 \
+ --no-deps --no-binary :all:
+
+ 2) Download SDK binary wheel file:
+ python pip -m download --dest /tmp/dir apache-beam==2.0.0 \
+ --no-deps --no-binary :all: --python-version 27 \
+ --implementation cp --abi cp27mu --platform manylinux1_x86_64
+ """
+ package_file = None
+ if len(args) >= 8:
+ # package_name==x.y.z
+ if '==' in args[6]:
+ distribution_name = args[6][0:args[6].find('==')]
+ distribution_version = args[6][args[6].find('==') + 2:]
+
+ if args[8] == '--no-binary':
+ package_file = '%s-%s.zip' % (
+ distribution_name, distribution_version)
+ elif args[8] == '--only-binary' and len(args) >= 18:
+ if not has_wheels:
+ # Imitate the case when desired wheel distribution is not in PyPI.
+ raise RuntimeError('No matching distribution.')
+
+ # Per PEP-0427 in wheel filenames non-alphanumeric characters
+ # in distribution name are replaced with underscore.
+ distribution_name = distribution_name.replace('-', '_')
+ if args[17] == 'manylinux2014_x86_64':
+ args[17] = 'manylinux_2_17_x86_64.' + args[17]
+ package_file = '%s-%s-%s%s-%s-%s.whl' % (
+ distribution_name,
+ distribution_version,
+ args[13], # implementation
+ args[11], # python version
+ args[15], # abi tag
+ args[17] # platform
+ )
+
+ assert package_file, 'Pip fake does not support the command: ' + str(args)
+ self.create_temp_file(
+ FileSystems.join(args[5], package_file), 'Package content.')
+
+ return pip_fake
+
@mock.patch('apache_beam.runners.portability.stager.open')
@mock.patch('apache_beam.runners.portability.stager.get_new_http')
def test_download_file_https(self, mock_new_http, mock_open):
@@ -375,10 +433,38 @@ class StagerTest(unittest.TestCase):
self.update_options(options)
options.view_as(SetupOptions).sdk_location = 'default'
- _, staged_resources = self.stager.create_and_stage_job_resources(
- options, temp_dir=self.make_temp_dir(), staging_location=staging_dir)
+ with mock.patch(
+ 'apache_beam.utils.processes.check_output',
+ self.build_fake_pip_download_command_handler(has_wheels=False)):
+ _, staged_resources = self.stager.create_and_stage_job_resources(
+ options, temp_dir=self.make_temp_dir(), staging_location=staging_dir)
+
+ self.assertEqual([names.STAGED_SDK_SOURCES_FILENAME], staged_resources)
+
+ with open(os.path.join(staging_dir,
+ names.STAGED_SDK_SOURCES_FILENAME)) as f:
+ self.assertEqual(f.read(), 'Package content.')
+
+ def test_sdk_location_default_with_wheels(self):
+ staging_dir = self.make_temp_dir()
+
+ options = PipelineOptions()
+ self.update_options(options)
+ options.view_as(SetupOptions).sdk_location = 'default'
- self.assertEqual([], staged_resources)
+ with mock.patch(
+ 'apache_beam.utils.processes.check_output',
+ self.build_fake_pip_download_command_handler(has_wheels=True)):
+ _, staged_resources = self.stager.create_and_stage_job_resources(
+ options, temp_dir=self.make_temp_dir(), staging_location=staging_dir)
+
+ self.assertEqual(len(staged_resources), 2)
+ self.assertEqual(staged_resources[0], names.STAGED_SDK_SOURCES_FILENAME)
+ # Exact name depends on the version of the SDK.
+ self.assertTrue(staged_resources[1].endswith('whl'))
+ for name in staged_resources:
+ with open(os.path.join(staging_dir, name)) as f:
+ self.assertEqual(f.read(), 'Package content.')
def test_sdk_location_local_directory(self):
staging_dir = self.make_temp_dir()
diff --git a/sdks/python/container/boot.go b/sdks/python/container/boot.go
index c67239f0564..dd74acabfd8 100644
--- a/sdks/python/container/boot.go
+++ b/sdks/python/container/boot.go
@@ -377,11 +377,6 @@ func installSetupPackages(files []string, workDir string, requirementsFiles []st
log.Printf("Failed to setup acceptable wheel specs, leave it as empty: %v", err)
}
- pkgName := "apache-beam"
- isSdkInstalled := isPackageInstalled(pkgName)
- if !isSdkInstalled {
- return fmt.Errorf("Apache Beam is not installed in the runtime environment. If you use a custom container image, you must install apache-beam package in the custom image using same version of Beam as in the pipeline submission environment. For more information, see: the https://beam.apache.org/documentation/runtime/environments/.")
- }
// Install the Dataflow Python SDK and worker packages.
// We install the extra requirements in case of using the beam sdk. These are ignored by pip
// if the user is using an SDK that does not provide these.
diff --git a/sdks/python/container/piputil.go b/sdks/python/container/piputil.go
index 720bf372c53..b4885e6356d 100644
--- a/sdks/python/container/piputil.go
+++ b/sdks/python/container/piputil.go
@@ -22,7 +22,6 @@ import (
"fmt"
"log"
"os"
- "os/exec"
"path/filepath"
"strings"
@@ -51,18 +50,6 @@ func pipInstallRequirements(files []string, dir, name string) error {
return nil
}
-// isPackageInstalled checks if the given package is installed in the
-// environment.
-func isPackageInstalled(pkgName string) bool {
- cmd := exec.Command("python", "-m", "pip", "show", pkgName)
- if err := cmd.Run(); err != nil {
- if _, ok := err.(*exec.ExitError); ok {
- return false
- }
- }
- return true
-}
-
// pipInstallPackage installs the given package, if present.
func pipInstallPackage(files []string, dir, name string, force, optional bool, extras []string) error {
for _, file := range files {