You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ro...@apache.org on 2017/05/11 21:11:30 UTC
[1/3] beam git commit: [BEAM-1345] Mark Pipeline as public.
Repository: beam
Updated Branches:
refs/heads/master 5eab219b7 -> b1c51ccbe
[BEAM-1345] Mark Pipeline as public.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/1b89697e
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/1b89697e
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/1b89697e
Branch: refs/heads/master
Commit: 1b89697e5f92bbea4a5d5a8f838409d41dcfceb4
Parents: 5eab219
Author: Robert Bradshaw <ro...@gmail.com>
Authored: Thu May 11 13:30:32 2017 -0700
Committer: Robert Bradshaw <ro...@gmail.com>
Committed: Thu May 11 14:06:21 2017 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/pipeline.py | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/1b89697e/sdks/python/apache_beam/pipeline.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/pipeline.py b/sdks/python/apache_beam/pipeline.py
index 83c7287..ec8dde4 100644
--- a/sdks/python/apache_beam/pipeline.py
+++ b/sdks/python/apache_beam/pipeline.py
@@ -67,6 +67,9 @@ from apache_beam.options.pipeline_options_validator import PipelineOptionsValida
from apache_beam.utils.annotations import deprecated
+__all__ = ['Pipeline']
+
+
class Pipeline(object):
"""A pipeline object that manages a DAG of PValues and their PTransforms.
@@ -182,6 +185,8 @@ class Pipeline(object):
def visit(self, visitor):
"""Visits depth-first every node of a pipeline's DAG.
+ Runner-internal implementation detail; no backwards-compatibility guarantees
+
Args:
visitor: PipelineVisitor object whose callbacks will be called for each
node visited. See PipelineVisitor comments.
@@ -333,6 +338,7 @@ class Pipeline(object):
return Visitor.ok
def to_runner_api(self):
+ """For internal use only; no backwards-compatibility guarantees."""
from apache_beam.runners import pipeline_context
from apache_beam.runners.api import beam_runner_api_pb2
context = pipeline_context.PipelineContext()
@@ -346,6 +352,7 @@ class Pipeline(object):
@staticmethod
def from_runner_api(proto, runner, options):
+ """For internal use only; no backwards-compatibility guarantees."""
p = Pipeline(runner=runner, options=options)
from apache_beam.runners import pipeline_context
context = pipeline_context.PipelineContext(proto.components)
[3/3] beam git commit: Closes #3096
Posted by ro...@apache.org.
Closes #3096
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/b1c51ccb
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/b1c51ccb
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/b1c51ccb
Branch: refs/heads/master
Commit: b1c51ccbe883fdc2c01f4ca85520938af9395314
Parents: 5eab219 c831734
Author: Robert Bradshaw <ro...@gmail.com>
Authored: Thu May 11 14:06:23 2017 -0700
Committer: Robert Bradshaw <ro...@gmail.com>
Committed: Thu May 11 14:06:23 2017 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/pipeline.py | 7 +++++++
sdks/python/apache_beam/runners/api/__init__.py | 4 +++-
sdks/python/apache_beam/runners/common.py | 5 ++++-
sdks/python/apache_beam/runners/dataflow/__init__.py | 9 +++++++++
sdks/python/apache_beam/runners/dataflow/dataflow_runner.py | 3 +++
.../apache_beam/runners/dataflow/test_dataflow_runner.py | 3 +++
sdks/python/apache_beam/runners/direct/__init__.py | 6 +++++-
sdks/python/apache_beam/runners/direct/direct_runner.py | 3 +++
sdks/python/apache_beam/runners/pipeline_context.py | 6 ++++++
sdks/python/apache_beam/runners/portability/__init__.py | 2 ++
sdks/python/apache_beam/runners/runner.py | 3 +++
sdks/python/apache_beam/runners/worker/__init__.py | 2 ++
12 files changed, 50 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[2/3] beam git commit: [BEAM-1345] Clearly delineate public api in
runners package.
Posted by ro...@apache.org.
[BEAM-1345] Clearly delineate public api in runners package.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/c8317345
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/c8317345
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/c8317345
Branch: refs/heads/master
Commit: c8317345ae469008f9ab371855e329d8b2eb32b7
Parents: 1b89697
Author: Robert Bradshaw <ro...@gmail.com>
Authored: Thu May 11 13:41:24 2017 -0700
Committer: Robert Bradshaw <ro...@gmail.com>
Committed: Thu May 11 14:06:22 2017 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/runners/api/__init__.py | 4 +++-
sdks/python/apache_beam/runners/common.py | 5 ++++-
sdks/python/apache_beam/runners/dataflow/__init__.py | 9 +++++++++
sdks/python/apache_beam/runners/dataflow/dataflow_runner.py | 3 +++
.../apache_beam/runners/dataflow/test_dataflow_runner.py | 3 +++
sdks/python/apache_beam/runners/direct/__init__.py | 6 +++++-
sdks/python/apache_beam/runners/direct/direct_runner.py | 3 +++
sdks/python/apache_beam/runners/pipeline_context.py | 6 ++++++
sdks/python/apache_beam/runners/portability/__init__.py | 2 ++
sdks/python/apache_beam/runners/runner.py | 3 +++
sdks/python/apache_beam/runners/worker/__init__.py | 2 ++
11 files changed, 43 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/c8317345/sdks/python/apache_beam/runners/api/__init__.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/api/__init__.py b/sdks/python/apache_beam/runners/api/__init__.py
index e94673c..bf95208 100644
--- a/sdks/python/apache_beam/runners/api/__init__.py
+++ b/sdks/python/apache_beam/runners/api/__init__.py
@@ -15,7 +15,9 @@
# limitations under the License.
#
-"""Checked in to avoid protoc dependency for Python development.
+"""For internal use only; no backwards-compatibility guarantees.
+
+Checked in to avoid protoc dependency for Python development.
Regenerate files with::
http://git-wip-us.apache.org/repos/asf/beam/blob/c8317345/sdks/python/apache_beam/runners/common.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/common.py b/sdks/python/apache_beam/runners/common.py
index 86db711..8453569 100644
--- a/sdks/python/apache_beam/runners/common.py
+++ b/sdks/python/apache_beam/runners/common.py
@@ -17,7 +17,10 @@
# cython: profile=True
-"""Worker operations executor."""
+"""Worker operations executor.
+
+For internal use only; no backwards-compatibility guarantees.
+"""
import sys
import traceback
http://git-wip-us.apache.org/repos/asf/beam/blob/c8317345/sdks/python/apache_beam/runners/dataflow/__init__.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/dataflow/__init__.py b/sdks/python/apache_beam/runners/dataflow/__init__.py
index cce3aca..6674ba5 100644
--- a/sdks/python/apache_beam/runners/dataflow/__init__.py
+++ b/sdks/python/apache_beam/runners/dataflow/__init__.py
@@ -14,3 +14,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+
+"""The DataflowRunner executes pipelines on Google Cloud Dataflow.
+
+Anything in this package not imported here is an internal implementation detail
+with no backwards-compatibility guarantees.
+"""
+
+from apache_beam.runners.dataflow.dataflow_runner import DataflowRunner
+from apache_beam.runners.dataflow.test_dataflow_runner import TestDataflowRunner
http://git-wip-us.apache.org/repos/asf/beam/blob/c8317345/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
index 796a67b..3d8437c 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
@@ -46,6 +46,9 @@ from apache_beam.typehints import typehints
from apache_beam.options.pipeline_options import StandardOptions
+__all__ = ['DataflowRunner']
+
+
class DataflowRunner(PipelineRunner):
"""A runner that creates job graphs and submits them for remote execution.
http://git-wip-us.apache.org/repos/asf/beam/blob/c8317345/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py
index 4fd1026..b339882 100644
--- a/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py
+++ b/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py
@@ -22,6 +22,9 @@ from apache_beam.options.pipeline_options import TestOptions, GoogleCloudOptions
from apache_beam.runners.dataflow.dataflow_runner import DataflowRunner
+__all__ = ['TestDataflowRunner']
+
+
class TestDataflowRunner(DataflowRunner):
def run(self, pipeline):
"""Execute test pipeline and verify test matcher"""
http://git-wip-us.apache.org/repos/asf/beam/blob/c8317345/sdks/python/apache_beam/runners/direct/__init__.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/direct/__init__.py b/sdks/python/apache_beam/runners/direct/__init__.py
index 0d64513..0f82756 100644
--- a/sdks/python/apache_beam/runners/direct/__init__.py
+++ b/sdks/python/apache_beam/runners/direct/__init__.py
@@ -15,5 +15,9 @@
# limitations under the License.
#
-"""Inprocess runner executes pipelines locally in a single process."""
+"""Inprocess runner executes pipelines locally in a single process.
+
+Anything in this package not imported here is an internal implementation detail
+with no backwards-compatibility guarantees.
+"""
from apache_beam.runners.direct.direct_runner import DirectRunner
http://git-wip-us.apache.org/repos/asf/beam/blob/c8317345/sdks/python/apache_beam/runners/direct/direct_runner.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/direct/direct_runner.py b/sdks/python/apache_beam/runners/direct/direct_runner.py
index 535aac3..ecf5114 100644
--- a/sdks/python/apache_beam/runners/direct/direct_runner.py
+++ b/sdks/python/apache_beam/runners/direct/direct_runner.py
@@ -36,6 +36,9 @@ from apache_beam.options.pipeline_options import DirectOptions
from apache_beam.options.value_provider import RuntimeValueProvider
+__all__ = ['DirectRunner']
+
+
class DirectRunner(PipelineRunner):
"""Executes a single pipeline on the local machine."""
http://git-wip-us.apache.org/repos/asf/beam/blob/c8317345/sdks/python/apache_beam/runners/pipeline_context.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/pipeline_context.py b/sdks/python/apache_beam/runners/pipeline_context.py
index d3d3c24..1c89d06 100644
--- a/sdks/python/apache_beam/runners/pipeline_context.py
+++ b/sdks/python/apache_beam/runners/pipeline_context.py
@@ -15,6 +15,12 @@
# limitations under the License.
#
+"""Utility class for serializing pipelines via the runner API.
+
+For internal use only; no backwards-compatibility guarantees.
+"""
+
+
from apache_beam import pipeline
from apache_beam import pvalue
from apache_beam import coders
http://git-wip-us.apache.org/repos/asf/beam/blob/c8317345/sdks/python/apache_beam/runners/portability/__init__.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/portability/__init__.py b/sdks/python/apache_beam/runners/portability/__init__.py
index cce3aca..7af93ed 100644
--- a/sdks/python/apache_beam/runners/portability/__init__.py
+++ b/sdks/python/apache_beam/runners/portability/__init__.py
@@ -14,3 +14,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+
+"""This runner is experimental; no backwards-compatibility guarantees."""
http://git-wip-us.apache.org/repos/asf/beam/blob/c8317345/sdks/python/apache_beam/runners/runner.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/runner.py b/sdks/python/apache_beam/runners/runner.py
index d875fdc..af00d8f 100644
--- a/sdks/python/apache_beam/runners/runner.py
+++ b/sdks/python/apache_beam/runners/runner.py
@@ -26,6 +26,9 @@ import shutil
import tempfile
+__all__ = ['PipelineRunner', 'PipelineState', 'PipelineResult']
+
+
def _get_runner_map(runner_names, module_path):
"""Create a map of runner name in lower case to full import path to the
runner class.
http://git-wip-us.apache.org/repos/asf/beam/blob/c8317345/sdks/python/apache_beam/runners/worker/__init__.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/worker/__init__.py b/sdks/python/apache_beam/runners/worker/__init__.py
index cce3aca..0bce5d6 100644
--- a/sdks/python/apache_beam/runners/worker/__init__.py
+++ b/sdks/python/apache_beam/runners/worker/__init__.py
@@ -14,3 +14,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+
+"""For internal use only; no backwards-compatibility guarantees."""