You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by al...@apache.org on 2017/06/29 19:46:09 UTC
[1/2] beam git commit: Select SDK distribution based on the selected
SDK name
Repository: beam
Updated Branches:
refs/heads/master 369007316 -> 99b5b0563
Select SDK distribution based on the selected SDK name
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/dc1dca86
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/dc1dca86
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/dc1dca86
Branch: refs/heads/master
Commit: dc1dca8633775545b5b4b509724716108d5d01e4
Parents: 3690073
Author: Ahmet Altay <al...@google.com>
Authored: Thu Jun 29 10:56:25 2017 -0700
Committer: Ahmet Altay <al...@google.com>
Committed: Thu Jun 29 12:45:41 2017 -0700
----------------------------------------------------------------------
.../runners/dataflow/internal/dependency.py | 23 ++++++++++++++------
1 file changed, 16 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/dc1dca86/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
index a40a582..62c09ed 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
@@ -69,12 +69,15 @@ from apache_beam.utils import processes
from apache_beam.options.pipeline_options import GoogleCloudOptions
from apache_beam.options.pipeline_options import SetupOptions
+# All constants are for internal use only; no backwards-compatibility
+# guarantees.
+# In a released version BEAM_CONTAINER_VERSION and BEAM_FNAPI_CONTAINER_VERSION
+# should match each other, and should be in the same format as the SDK version
+# (i.e. MAJOR.MINOR.PATCH). For non-released (dev) versions, read below.
# Update this version to the next version whenever there is a change that will
# require changes to legacy Dataflow worker execution environment.
# This should be in the beam-[version]-[date] format, date is optional.
-# BEAM_CONTAINER_VERSION and BEAM_FNAPI_CONTAINER version should coincide
-# when we make a release.
BEAM_CONTAINER_VERSION = 'beam-2.1.0-20170626'
# Update this version to the next version whenever there is a change that
# requires changes to SDK harness container or SDK harness launcher.
@@ -86,9 +89,14 @@ WORKFLOW_TARBALL_FILE = 'workflow.tar.gz'
REQUIREMENTS_FILE = 'requirements.txt'
EXTRA_PACKAGES_FILE = 'extra_packages.txt'
+# Package names for different distributions
GOOGLE_PACKAGE_NAME = 'google-cloud-dataflow'
BEAM_PACKAGE_NAME = 'apache-beam'
+# SDK identifiers for different distributions
+GOOGLE_SDK_NAME = 'Google Cloud Dataflow SDK for Python'
+BEAM_SDK_NAME = 'Apache Beam SDK for Python'
+
def _dependency_file_copy(from_path, to_path):
"""Copies a local file to a GCS file or vice versa."""
@@ -536,19 +544,20 @@ def get_sdk_name_and_version():
container_version = _get_required_container_version()
try:
pkg.get_distribution(GOOGLE_PACKAGE_NAME)
- return ('Google Cloud Dataflow SDK for Python', container_version)
+ return (GOOGLE_SDK_NAME, container_version)
except pkg.DistributionNotFound:
- return ('Apache Beam SDK for Python', beam_version.__version__)
+ return (BEAM_SDK_NAME, beam_version.__version__)
def get_sdk_package_name():
"""For internal use only; no backwards-compatibility guarantees.
Returns the PyPI package name to be staged to Google Cloud Dataflow."""
- container_version = _get_required_container_version()
- if container_version == BEAM_CONTAINER_VERSION:
+ sdk_name, _ = get_sdk_name_and_version()
+ if sdk_name == GOOGLE_SDK_NAME:
+ return GOOGLE_PACKAGE_NAME
+ else:
return BEAM_PACKAGE_NAME
- return GOOGLE_PACKAGE_NAME
def _download_pypi_sdk_package(temp_dir):
[2/2] beam git commit: This closes #3474
Posted by al...@apache.org.
This closes #3474
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/99b5b056
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/99b5b056
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/99b5b056
Branch: refs/heads/master
Commit: 99b5b0563f85e6e2e22bb7b9148b4058955aeae1
Parents: 3690073 dc1dca8
Author: Ahmet Altay <al...@google.com>
Authored: Thu Jun 29 12:45:44 2017 -0700
Committer: Ahmet Altay <al...@google.com>
Committed: Thu Jun 29 12:45:44 2017 -0700
----------------------------------------------------------------------
.../runners/dataflow/internal/dependency.py | 23 ++++++++++++++------
1 file changed, 16 insertions(+), 7 deletions(-)
----------------------------------------------------------------------