You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2016/07/11 20:15:04 UTC
[1/2] incubator-beam git commit: Allow ".tar" files in extra_packages
Repository: incubator-beam
Updated Branches:
refs/heads/python-sdk 1b1c8d538 -> 39dda320a
Allow ".tar" files in extra_packages
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/516f15dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/516f15dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/516f15dd
Branch: refs/heads/python-sdk
Commit: 516f15dd656fd9a1df36c91a25e555ca1f7e1e42
Parents: 1b1c8d5
Author: Charles Chen <cc...@google.com>
Authored: Sun Jul 10 20:59:08 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Mon Jul 11 13:14:12 2016 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/utils/dependency.py | 8 ++++----
sdks/python/apache_beam/utils/dependency_test.py | 12 ++++++++----
2 files changed, 12 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/516f15dd/sdks/python/apache_beam/utils/dependency.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/utils/dependency.py b/sdks/python/apache_beam/utils/dependency.py
index ddb640a..b809cf2 100644
--- a/sdks/python/apache_beam/utils/dependency.py
+++ b/sdks/python/apache_beam/utils/dependency.py
@@ -29,7 +29,7 @@ and a --setup_file option.
If --setup_file is present then it is assumed that the folder containing the
file specified by the option has the typical layout required by setuptools and
it will run 'python setup.py sdist' to produce a source distribution. The
-resulting tarball (a file ending in .tar.gz) will be staged at the GCS staging
+resulting tarball (a .tar or .tar.gz file) will be staged at the GCS staging
location specified as job option. When a worker starts it will check for the
presence of this file and will run 'easy_install tarball' to install the
package in the worker.
@@ -137,10 +137,11 @@ def _stage_extra_packages(extra_packages, staging_location, temp_dir,
staging_temp_dir = None
local_packages = []
for package in extra_packages:
- if not os.path.basename(package).endswith('.tar.gz'):
+ if not (os.path.basename(package).endswith('.tar') or
+ os.path.basename(package).endswith('.tar.gz')):
raise RuntimeError(
'The --extra_packages option expects a full path ending with '
- '\'.tar.gz\' instead of %s' % package)
+ '\'.tar\' or \'.tar.gz\' instead of %s' % package)
if not os.path.isfile(package):
if package.startswith('gs://'):
@@ -460,4 +461,3 @@ def _download_pypi_sdk_package(temp_dir):
'Failed to download a source distribution for the running SDK. Expected '
'either %s or %s to be found in the download folder.' % (
zip_expected, tgz_expected))
-
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/516f15dd/sdks/python/apache_beam/utils/dependency_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/utils/dependency_test.py b/sdks/python/apache_beam/utils/dependency_test.py
index ab6446d..b7a9296 100644
--- a/sdks/python/apache_beam/utils/dependency_test.py
+++ b/sdks/python/apache_beam/utils/dependency_test.py
@@ -333,6 +333,8 @@ class SetupTest(unittest.TestCase):
self.create_temp_file(
os.path.join(source_dir, 'xyz.tar.gz'), 'nothing')
self.create_temp_file(
+ os.path.join(source_dir, 'xyz2.tar'), 'nothing')
+ self.create_temp_file(
os.path.join(source_dir, dependency.EXTRA_PACKAGES_FILE), 'nothing')
options = PipelineOptions()
@@ -341,6 +343,7 @@ class SetupTest(unittest.TestCase):
options.view_as(SetupOptions).extra_packages = [
os.path.join(source_dir, 'abc.tar.gz'),
os.path.join(source_dir, 'xyz.tar.gz'),
+ os.path.join(source_dir, 'xyz2.tar'),
'gs://my-gcs-bucket/gcs.tar.gz']
gcs_copied_files = []
@@ -359,13 +362,13 @@ class SetupTest(unittest.TestCase):
dependency._dependency_file_copy = file_copy
self.assertEqual(
- ['abc.tar.gz', 'xyz.tar.gz', 'gcs.tar.gz',
+ ['abc.tar.gz', 'xyz.tar.gz', 'xyz2.tar', 'gcs.tar.gz',
dependency.EXTRA_PACKAGES_FILE,
names.PICKLED_MAIN_SESSION_FILE],
dependency.stage_job_resources(options))
with open(os.path.join(staging_dir, dependency.EXTRA_PACKAGES_FILE)) as f:
- self.assertEqual(['abc.tar.gz\n', 'xyz.tar.gz\n', 'gcs.tar.gz\n'],
- f.readlines())
+ self.assertEqual(['abc.tar.gz\n', 'xyz.tar.gz\n', 'xyz2.tar\n',
+ 'gcs.tar.gz\n'], f.readlines())
self.assertEqual(['gs://my-gcs-bucket/gcs.tar.gz'], gcs_copied_files)
def test_with_extra_packages_missing_files(self):
@@ -398,7 +401,8 @@ class SetupTest(unittest.TestCase):
self.assertEqual(
cm.exception.message,
'The --extra_packages option expects a full path ending with '
- '\'.tar.gz\' instead of %s' % os.path.join(source_dir, 'abc.tgz'))
+ '\'.tar\' or \'.tar.gz\' instead of %s' % os.path.join(source_dir,
+ 'abc.tgz'))
if __name__ == '__main__':
[2/2] incubator-beam git commit: Closes #621
Posted by dh...@apache.org.
Closes #621
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/39dda320
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/39dda320
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/39dda320
Branch: refs/heads/python-sdk
Commit: 39dda320a02fb706e66b918228a12b23c5124cbd
Parents: 1b1c8d5 516f15d
Author: Dan Halperin <dh...@google.com>
Authored: Mon Jul 11 13:14:13 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Mon Jul 11 13:14:13 2016 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/utils/dependency.py | 8 ++++----
sdks/python/apache_beam/utils/dependency_test.py | 12 ++++++++----
2 files changed, 12 insertions(+), 8 deletions(-)
----------------------------------------------------------------------