You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ro...@apache.org on 2016/09/26 22:05:39 UTC
[1/2] incubator-beam git commit: Allow .whl files to be staged with
--extra_package
Repository: incubator-beam
Updated Branches:
refs/heads/python-sdk 8a333a661 -> 19e3eff91
Allow .whl files to be staged with --extra_package
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/71c474a5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/71c474a5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/71c474a5
Branch: refs/heads/python-sdk
Commit: 71c474a5d7227be9391388fda41875305b952f93
Parents: 8a333a6
Author: Charles Chen <cc...@google.com>
Authored: Thu Sep 22 20:19:05 2016 -0700
Committer: Robert Bradshaw <ro...@google.com>
Committed: Mon Sep 26 15:05:00 2016 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/utils/dependency.py | 14 +++++++++++---
sdks/python/apache_beam/utils/dependency_test.py | 12 +++++++-----
sdks/python/apache_beam/utils/options.py | 16 ++++++++--------
3 files changed, 26 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/71c474a5/sdks/python/apache_beam/utils/dependency.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/utils/dependency.py b/sdks/python/apache_beam/utils/dependency.py
index 314bd1b..ab311a0 100644
--- a/sdks/python/apache_beam/utils/dependency.py
+++ b/sdks/python/apache_beam/utils/dependency.py
@@ -141,10 +141,18 @@ def _stage_extra_packages(extra_packages, staging_location, temp_dir,
local_packages = []
for package in extra_packages:
if not (os.path.basename(package).endswith('.tar') or
- os.path.basename(package).endswith('.tar.gz')):
+ os.path.basename(package).endswith('.tar.gz') or
+ os.path.basename(package).endswith('.whl')):
raise RuntimeError(
- 'The --extra_packages option expects a full path ending with '
- '\'.tar\' or \'.tar.gz\' instead of %s' % package)
+ 'The --extra_package option expects a full path ending with '
+ '".tar" or ".tar.gz" instead of %s' % package)
+ if os.path.basename(package).endswith('.whl'):
+ logging.warning(
+ 'The .whl package "%s" is provided in --extra_package. '
+ 'This functionality is not officially supported. Since wheel '
+ 'packages are binary distributions, this package must be '
+ 'binary-compatible with the worker environment (e.g. Python 2.7 '
+ 'running on an x64 Linux host).')
if not os.path.isfile(package):
if package.startswith('gs://'):
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/71c474a5/sdks/python/apache_beam/utils/dependency_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/utils/dependency_test.py b/sdks/python/apache_beam/utils/dependency_test.py
index ca31806..3549a07 100644
--- a/sdks/python/apache_beam/utils/dependency_test.py
+++ b/sdks/python/apache_beam/utils/dependency_test.py
@@ -342,6 +342,8 @@ class SetupTest(unittest.TestCase):
self.create_temp_file(
os.path.join(source_dir, 'xyz2.tar'), 'nothing')
self.create_temp_file(
+ os.path.join(source_dir, 'whl.whl'), 'nothing')
+ self.create_temp_file(
os.path.join(source_dir, dependency.EXTRA_PACKAGES_FILE), 'nothing')
options = PipelineOptions()
@@ -351,6 +353,7 @@ class SetupTest(unittest.TestCase):
os.path.join(source_dir, 'abc.tar.gz'),
os.path.join(source_dir, 'xyz.tar.gz'),
os.path.join(source_dir, 'xyz2.tar'),
+ os.path.join(source_dir, 'whl.whl'),
'gs://my-gcs-bucket/gcs.tar.gz']
gcs_copied_files = []
@@ -369,12 +372,12 @@ class SetupTest(unittest.TestCase):
dependency._dependency_file_copy = file_copy
self.assertEqual(
- ['abc.tar.gz', 'xyz.tar.gz', 'xyz2.tar', 'gcs.tar.gz',
+ ['abc.tar.gz', 'xyz.tar.gz', 'xyz2.tar', 'whl.whl', 'gcs.tar.gz',
dependency.EXTRA_PACKAGES_FILE],
dependency.stage_job_resources(options))
with open(os.path.join(staging_dir, dependency.EXTRA_PACKAGES_FILE)) as f:
self.assertEqual(['abc.tar.gz\n', 'xyz.tar.gz\n', 'xyz2.tar\n',
- 'gcs.tar.gz\n'], f.readlines())
+ 'whl.whl\n', 'gcs.tar.gz\n'], f.readlines())
self.assertEqual(['gs://my-gcs-bucket/gcs.tar.gz'], gcs_copied_files)
def test_with_extra_packages_missing_files(self):
@@ -406,9 +409,8 @@ class SetupTest(unittest.TestCase):
dependency.stage_job_resources(options)
self.assertEqual(
cm.exception.message,
- 'The --extra_packages option expects a full path ending with '
- '\'.tar\' or \'.tar.gz\' instead of %s' % os.path.join(source_dir,
- 'abc.tgz'))
+ 'The --extra_package option expects a full path ending with ".tar" or '
+ '".tar.gz" instead of %s' % os.path.join(source_dir, 'abc.tgz'))
if __name__ == '__main__':
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/71c474a5/sdks/python/apache_beam/utils/options.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/utils/options.py b/sdks/python/apache_beam/utils/options.py
index 700c080..a964036 100644
--- a/sdks/python/apache_beam/utils/options.py
+++ b/sdks/python/apache_beam/utils/options.py
@@ -442,14 +442,14 @@ class SetupOptions(PipelineOptions):
action='append',
default=None,
help=
- ('Local path to a Python package file. The file is expected to be a '
- 'compressed tarball with the suffix \'.tar.gz\' which can be '
- 'installed using the easy_install command of the standard setuptools '
- 'package. Multiple --extra_package options can be specified if more '
- 'than one package is needed. During job submission the files will be '
- 'staged in the staging area (--staging_location option) and the '
- 'workers will install them in same order they were specified on the '
- 'command line.'))
+ ('Local path to a Python package file. The file is expected to be (1) '
+ 'a package tarball (".tar") or (2) a compressed package tarball '
+ '(".tar.gz") which can be installed using the "pip install" command '
+ 'of the standard pip package. Multiple --extra_package options can '
+ 'be specified if more than one package is needed. During job '
+ 'submission, the files will be staged in the staging area '
+ '(--staging_location option) and the workers will install them in '
+ 'same order they were specified on the command line.'))
# TODO(silviuc): Add --files_to_stage option.
# This could potentially replace the --requirements_file and --setup_file.
[2/2] incubator-beam git commit: Closes #991
Posted by ro...@apache.org.
Closes #991
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/19e3eff9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/19e3eff9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/19e3eff9
Branch: refs/heads/python-sdk
Commit: 19e3eff9153e1c5f57964c3256f9adcadf8a5c03
Parents: 8a333a6 71c474a
Author: Robert Bradshaw <ro...@google.com>
Authored: Mon Sep 26 15:05:19 2016 -0700
Committer: Robert Bradshaw <ro...@google.com>
Committed: Mon Sep 26 15:05:19 2016 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/utils/dependency.py | 14 +++++++++++---
sdks/python/apache_beam/utils/dependency_test.py | 12 +++++++-----
sdks/python/apache_beam/utils/options.py | 16 ++++++++--------
3 files changed, 26 insertions(+), 16 deletions(-)
----------------------------------------------------------------------