You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ro...@apache.org on 2016/07/01 00:01:58 UTC
[2/4] incubator-beam git commit: Define GOOGLE_PACKAGE_NAME and use
it everywhere
Define GOOGLE_PACKAGE_NAME and use it everywhere
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/33720ec4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/33720ec4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/33720ec4
Branch: refs/heads/python-sdk
Commit: 33720ec4f0ba4c002692674976cbdd5d5bdf0529
Parents: 0bda677
Author: Silviu Calinoiu <si...@google.com>
Authored: Thu Jun 30 13:46:25 2016 -0700
Committer: Robert Bradshaw <ro...@google.com>
Committed: Thu Jun 30 17:00:11 2016 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/utils/dependency.py | 19 ++++++++-----------
1 file changed, 8 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/33720ec4/sdks/python/apache_beam/utils/dependency.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/utils/dependency.py b/sdks/python/apache_beam/utils/dependency.py
index be7cd03..c52a5b7 100644
--- a/sdks/python/apache_beam/utils/dependency.py
+++ b/sdks/python/apache_beam/utils/dependency.py
@@ -72,6 +72,8 @@ WORKFLOW_TARBALL_FILE = 'workflow.tar.gz'
REQUIREMENTS_FILE = 'requirements.txt'
EXTRA_PACKAGES_FILE = 'extra_packages.txt'
+GOOGLE_PACKAGE_NAME = 'google-cloud-dataflow'
+
def _dependency_file_copy(from_path, to_path):
"""Copies a local file to a GCS file or vice versa."""
@@ -431,16 +433,11 @@ def _stage_dataflow_sdk_tarball(sdk_remote_location, staged_path, temp_dir):
def get_required_container_version():
"""Returns the Google Cloud Dataflow container version for remote execution.
-
- Raises:
- pkg_resources.DistributionNotFound: if one of the expected package names
- are not found: 'google-cloud-dataflow' (right now) and 'apache-beam'
- (in the future).
"""
# TODO(silviuc): Handle apache-beam versions when we have official releases.
import pkg_resources as pkg
try:
- version = pkg.get_distribution('google-cloud-dataflow').version
+ version = pkg.get_distribution(GOOGLE_PACKAGE_NAME).version
# We drop any pre/post parts of the version and we keep only the X.Y.Z format.
# For instance the 0.3.0rc2 SDK version translates into 0.3.0.
return '%s.%s.%s' % pkg.parse_version(version)._version.release
@@ -453,13 +450,12 @@ def get_required_container_version():
def _download_pypi_sdk_package(temp_dir):
"""Downloads SDK package from PyPI and returns path to local path."""
# TODO(silviuc): Handle apache-beam versions when we have official releases.
- PACKAGE_NAME = 'google-cloud-dataflow'
import pkg_resources as pkg
- version = pkg.get_distribution('google-cloud-dataflow').version
+ version = pkg.get_distribution(GOOGLE_PACKAGE_NAME).version
# Get a source distribution for the SDK package from PyPI.
cmd_args = [
'pip', 'install', '--download', temp_dir,
- '%s==%s' % (PACKAGE_NAME, version),
+ '%s==%s' % (GOOGLE_PACKAGE_NAME, version),
'--no-binary', ':all:', '--no-deps']
logging.info('Executing command: %s', cmd_args)
result = processes.call(cmd_args)
@@ -467,11 +463,12 @@ def _download_pypi_sdk_package(temp_dir):
raise RuntimeError(
'Failed to execute command: %s. Exit code %d',
cmd_args, result)
- zip_expected = os.path.join(temp_dir, '%s-%s.zip' % (PACKAGE_NAME, version))
+ zip_expected = os.path.join(
+ temp_dir, '%s-%s.zip' % (GOOGLE_PACKAGE_NAME, version))
if os.path.exists(zip_expected):
return zip_expected
tgz_expected = os.path.join(
- temp_dir, '%s-%s.tar.gz' % (PACKAGE_NAME, version))
+ temp_dir, '%s-%s.tar.gz' % (GOOGLE_PACKAGE_NAME, version))
if os.path.exists(tgz_expected):
return tgz_expected
raise RuntimeError(