You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by al...@apache.org on 2017/10/25 23:33:36 UTC
[1/2] beam git commit: Add zip to the list of accepted extra package
file types.
Repository: beam
Updated Branches:
refs/heads/master 94d7f747b -> d5aff5dad
Add zip to the list of accepted extra package file types.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/41417f77
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/41417f77
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/41417f77
Branch: refs/heads/master
Commit: 41417f778fc3f62f3a27ddbb5c1fb717cc25f7b6
Parents: 94d7f74
Author: Ahmet Altay <al...@google.com>
Authored: Fri Oct 20 14:02:48 2017 -0700
Committer: Ahmet Altay <al...@google.com>
Committed: Wed Oct 25 16:33:19 2017 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/options/pipeline_options.py | 11 ++++++-----
.../apache_beam/runners/dataflow/internal/dependency.py | 5 +++--
.../runners/dataflow/internal/dependency_test.py | 5 +++--
3 files changed, 12 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/41417f77/sdks/python/apache_beam/options/pipeline_options.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/options/pipeline_options.py b/sdks/python/apache_beam/options/pipeline_options.py
index a09c7c3..5278b8a 100644
--- a/sdks/python/apache_beam/options/pipeline_options.py
+++ b/sdks/python/apache_beam/options/pipeline_options.py
@@ -602,11 +602,12 @@ class SetupOptions(PipelineOptions):
default=None,
help=
('Local path to a Python package file. The file is expected to be (1) '
- 'a package tarball (".tar") or (2) a compressed package tarball '
- '(".tar.gz") which can be installed using the "pip install" command '
- 'of the standard pip package. Multiple --extra_package options can '
- 'be specified if more than one package is needed. During job '
- 'submission, the files will be staged in the staging area '
+ 'a package tarball (".tar"), (2) a compressed package tarball '
+ '(".tar.gz"), (3) a Wheel file (".whl") or (4) a compressed package '
+ 'zip file (".zip") which can be installed using the "pip install" '
+ 'command of the standard pip package. Multiple --extra_package '
+ 'options can be specified if more than one package is needed. During '
+ 'job submission, the files will be staged in the staging area '
'(--staging_location option) and the workers will install them in '
'same order they were specified on the command line.'))
http://git-wip-us.apache.org/repos/asf/beam/blob/41417f77/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
index c1edf7d..fba2df2 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
@@ -179,10 +179,11 @@ def _stage_extra_packages(extra_packages, staging_location, temp_dir,
for package in extra_packages:
if not (os.path.basename(package).endswith('.tar') or
os.path.basename(package).endswith('.tar.gz') or
- os.path.basename(package).endswith('.whl')):
+ os.path.basename(package).endswith('.whl') or
+ os.path.basename(package).endswith('.zip')):
raise RuntimeError(
'The --extra_package option expects a full path ending with '
- '".tar" or ".tar.gz" instead of %s' % package)
+ '".tar", ".tar.gz", ".whl" or ".zip" instead of %s' % package)
if os.path.basename(package).endswith('.whl'):
logging.warning(
'The .whl package "%s" is provided in --extra_package. '
http://git-wip-us.apache.org/repos/asf/beam/blob/41417f77/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py
index 6d9b061..f0e59bc 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py
@@ -427,8 +427,9 @@ class SetupTest(unittest.TestCase):
dependency.stage_job_resources(options)
self.assertEqual(
cm.exception.message,
- 'The --extra_package option expects a full path ending with ".tar" or '
- '".tar.gz" instead of %s' % os.path.join(source_dir, 'abc.tgz'))
+ 'The --extra_package option expects a full path ending with '
+ '".tar", ".tar.gz", ".whl" or ".zip" '
+ 'instead of %s' % os.path.join(source_dir, 'abc.tgz'))
if __name__ == '__main__':
[2/2] beam git commit: This closes #4022
Posted by al...@apache.org.
This closes #4022
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/d5aff5da
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/d5aff5da
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/d5aff5da
Branch: refs/heads/master
Commit: d5aff5dadca98a09533e2555ae25b5cad0dabba6
Parents: 94d7f74 41417f7
Author: Ahmet Altay <al...@google.com>
Authored: Wed Oct 25 16:33:27 2017 -0700
Committer: Ahmet Altay <al...@google.com>
Committed: Wed Oct 25 16:33:27 2017 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/options/pipeline_options.py | 11 ++++++-----
.../apache_beam/runners/dataflow/internal/dependency.py | 5 +++--
.../runners/dataflow/internal/dependency_test.py | 5 +++--
3 files changed, 12 insertions(+), 9 deletions(-)
----------------------------------------------------------------------