You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ro...@apache.org on 2017/05/11 21:29:36 UTC
[1/2] beam git commit: [BEAM-1340] Adds __all__ tags to classes in
package apache_beam/io.
Repository: beam
Updated Branches:
refs/heads/master 47cbabbd2 -> 0cc6dd727
[BEAM-1340] Adds __all__ tags to classes in package apache_beam/io.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/b8196295
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/b8196295
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/b8196295
Branch: refs/heads/master
Commit: b8196295700629121b295ec31b1c8de0284c62c9
Parents: 47cbabb
Author: chamikara@google.com <ch...@google.com>
Authored: Thu May 11 11:46:46 2017 -0700
Committer: Robert Bradshaw <ro...@gmail.com>
Committed: Thu May 11 14:25:41 2017 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/io/concat_source.py | 4 +++-
sdks/python/apache_beam/io/filebasedsource.py | 2 ++
sdks/python/apache_beam/io/fileio.py | 7 +++++++
sdks/python/apache_beam/io/filesystem.py | 3 +++
sdks/python/apache_beam/io/filesystems.py | 2 ++
sdks/python/apache_beam/io/gcp/gcsfilesystem.py | 2 ++
sdks/python/apache_beam/io/gcp/gcsio.py | 3 +++
sdks/python/apache_beam/io/gcp/pubsub.py | 2 ++
sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py | 3 +++
sdks/python/apache_beam/io/iobase.py | 2 ++
sdks/python/apache_beam/io/localfilesystem.py | 2 ++
sdks/python/apache_beam/io/range_trackers.py | 7 ++++++-
sdks/python/apache_beam/io/source_test_utils.py | 8 ++++++++
13 files changed, 45 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/b8196295/sdks/python/apache_beam/io/concat_source.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/concat_source.py b/sdks/python/apache_beam/io/concat_source.py
index dfd1695..56c4cca 100644
--- a/sdks/python/apache_beam/io/concat_source.py
+++ b/sdks/python/apache_beam/io/concat_source.py
@@ -15,7 +15,9 @@
# limitations under the License.
#
-"""Concat Source, which reads the union of several other sources.
+"""For internal use only; no backwards-compatibility guarantees.
+
+Concat Source, which reads the union of several other sources.
"""
import bisect
http://git-wip-us.apache.org/repos/asf/beam/blob/b8196295/sdks/python/apache_beam/io/filebasedsource.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/filebasedsource.py b/sdks/python/apache_beam/io/filebasedsource.py
index 215e015..bb9efc4 100644
--- a/sdks/python/apache_beam/io/filebasedsource.py
+++ b/sdks/python/apache_beam/io/filebasedsource.py
@@ -38,6 +38,8 @@ from apache_beam.options.value_provider import check_accessible
MAX_NUM_THREADS_FOR_SIZE_ESTIMATION = 25
+__all__ = ['FileBasedSource']
+
class FileBasedSource(iobase.BoundedSource):
"""A ``BoundedSource`` for reading a file glob of a given type."""
http://git-wip-us.apache.org/repos/asf/beam/blob/b8196295/sdks/python/apache_beam/io/fileio.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/fileio.py
index ca3a759..aa18093 100644
--- a/sdks/python/apache_beam/io/fileio.py
+++ b/sdks/python/apache_beam/io/fileio.py
@@ -37,6 +37,8 @@ from apache_beam.options.value_provider import check_accessible
DEFAULT_SHARD_NAME_TEMPLATE = '-SSSSS-of-NNNNN'
+__all__ = ['FileBasedSink']
+
class FileSink(iobase.Sink):
"""A sink to a GCS or local files.
@@ -280,6 +282,11 @@ class FileSink(iobase.Sink):
return type(self) == type(other) and self.__dict__ == other.__dict__
+# Using FileBasedSink for the public API to be symmetric with FileBasedSource.
+# TODO: move code from FileSink to here and delete that class.
+FileBasedSink = FileSink
+
+
class FileSinkWriter(iobase.Writer):
"""The writer for FileSink.
"""
http://git-wip-us.apache.org/repos/asf/beam/blob/b8196295/sdks/python/apache_beam/io/filesystem.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/filesystem.py b/sdks/python/apache_beam/io/filesystem.py
index 3d35f3e..db6a1d0 100644
--- a/sdks/python/apache_beam/io/filesystem.py
+++ b/sdks/python/apache_beam/io/filesystem.py
@@ -30,6 +30,9 @@ logger = logging.getLogger(__name__)
DEFAULT_READ_BUFFER_SIZE = 16 * 1024 * 1024
+__all__ = ['CompressionTypes', 'CompressedFile', 'FileMetadata', 'FileSystem',
+ 'MatchResult']
+
class CompressionTypes(object):
"""Enum-like class representing known compression types."""
http://git-wip-us.apache.org/repos/asf/beam/blob/b8196295/sdks/python/apache_beam/io/filesystems.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/filesystems.py b/sdks/python/apache_beam/io/filesystems.py
index 29f0644..e039686 100644
--- a/sdks/python/apache_beam/io/filesystems.py
+++ b/sdks/python/apache_beam/io/filesystems.py
@@ -33,6 +33,8 @@ except ImportError:
pass
# pylint: enable=wrong-import-position, unused-import
+__all__ = ['FileSystems']
+
class FileSystems(object):
"""A class that defines the functions that can be performed on a filesystem.
http://git-wip-us.apache.org/repos/asf/beam/blob/b8196295/sdks/python/apache_beam/io/gcp/gcsfilesystem.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/gcp/gcsfilesystem.py b/sdks/python/apache_beam/io/gcp/gcsfilesystem.py
index dc71fce..ce8b5e6 100644
--- a/sdks/python/apache_beam/io/gcp/gcsfilesystem.py
+++ b/sdks/python/apache_beam/io/gcp/gcsfilesystem.py
@@ -26,6 +26,8 @@ from apache_beam.io.filesystem import FileSystem
from apache_beam.io.filesystem import MatchResult
from apache_beam.io.gcp import gcsio
+__all__ = ['GCSFileSystem']
+
class GCSFileSystem(FileSystem):
"""A GCS ``FileSystem`` implementation for accessing files on GCS.
http://git-wip-us.apache.org/repos/asf/beam/blob/b8196295/sdks/python/apache_beam/io/gcp/gcsio.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/gcp/gcsio.py b/sdks/python/apache_beam/io/gcp/gcsio.py
index c76c99d..774ee54 100644
--- a/sdks/python/apache_beam/io/gcp/gcsio.py
+++ b/sdks/python/apache_beam/io/gcp/gcsio.py
@@ -34,6 +34,9 @@ import traceback
from apache_beam.utils import retry
+__all__ = ['GcsIO']
+
+
# Issue a friendlier error message if the storage library is not available.
# TODO(silviuc): Remove this guard when storage is available everywhere.
try:
http://git-wip-us.apache.org/repos/asf/beam/blob/b8196295/sdks/python/apache_beam/io/gcp/pubsub.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/gcp/pubsub.py b/sdks/python/apache_beam/io/gcp/pubsub.py
index efc628d..103fce0 100644
--- a/sdks/python/apache_beam/io/gcp/pubsub.py
+++ b/sdks/python/apache_beam/io/gcp/pubsub.py
@@ -26,6 +26,8 @@ from apache_beam import coders
from apache_beam.runners.dataflow.native_io import iobase as dataflow_io
from apache_beam.transforms.display import DisplayDataItem
+__all__ = ['PubSubSink', 'PubSubSource']
+
class PubSubSource(dataflow_io.NativeSource):
"""Source for reading from a given Cloud Pub/Sub topic.
http://git-wip-us.apache.org/repos/asf/beam/blob/b8196295/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py b/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py
index f42b70f..844cbc5 100644
--- a/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py
+++ b/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py
@@ -24,6 +24,9 @@ from hamcrest.core.base_matcher import BaseMatcher
from apache_beam.testing.test_utils import compute_hash
from apache_beam.utils import retry
+__all__ = ['BigqueryMatcher']
+
+
# Protect against environments where bigquery library is not available.
# pylint: disable=wrong-import-order, wrong-import-position
try:
http://git-wip-us.apache.org/repos/asf/beam/blob/b8196295/sdks/python/apache_beam/io/iobase.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/iobase.py b/sdks/python/apache_beam/io/iobase.py
index e8ffb72..a80b12f 100644
--- a/sdks/python/apache_beam/io/iobase.py
+++ b/sdks/python/apache_beam/io/iobase.py
@@ -46,6 +46,8 @@ from apache_beam.transforms.display import HasDisplayData
from apache_beam.transforms.display import DisplayDataItem
from apache_beam.utils.windowed_value import WindowedValue
+__all__ = ['BoundedSource', 'RangeTracker', 'Read', 'Sink', 'Write', 'Writer']
+
# Encapsulates information about a bundle of a source generated when method
# BoundedSource.split() is invoked.
http://git-wip-us.apache.org/repos/asf/beam/blob/b8196295/sdks/python/apache_beam/io/localfilesystem.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/localfilesystem.py b/sdks/python/apache_beam/io/localfilesystem.py
index c670704..b08ac49 100644
--- a/sdks/python/apache_beam/io/localfilesystem.py
+++ b/sdks/python/apache_beam/io/localfilesystem.py
@@ -29,6 +29,8 @@ from apache_beam.io.filesystem import FileMetadata
from apache_beam.io.filesystem import FileSystem
from apache_beam.io.filesystem import MatchResult
+__all__ = ['LocalFileSystem']
+
class LocalFileSystem(FileSystem):
"""A Local ``FileSystem`` implementation for accessing files on disk.
http://git-wip-us.apache.org/repos/asf/beam/blob/b8196295/sdks/python/apache_beam/io/range_trackers.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/range_trackers.py b/sdks/python/apache_beam/io/range_trackers.py
index 000df81..9cb36e7 100644
--- a/sdks/python/apache_beam/io/range_trackers.py
+++ b/sdks/python/apache_beam/io/range_trackers.py
@@ -24,6 +24,9 @@ import threading
from apache_beam.io import iobase
+__all__ = ['OffsetRangeTracker', 'LexicographicKeyRangeTracker',
+ 'OrderedPositionRangeTracker', 'UnsplittableRangeTracker']
+
class OffsetRangeTracker(iobase.RangeTracker):
"""A 'RangeTracker' for non-negative positions of type 'long'."""
@@ -191,7 +194,9 @@ class OffsetRangeTracker(iobase.RangeTracker):
class GroupedShuffleRangeTracker(iobase.RangeTracker):
- """A 'RangeTracker' for positions used by'GroupedShuffleReader'.
+ """For internal use only; no backwards-compatibility guarantees.
+
+ A 'RangeTracker' for positions used by'GroupedShuffleReader'.
These positions roughly correspond to hashes of keys. In case of hash
collisions, multiple groups can have the same position. In that case, the
http://git-wip-us.apache.org/repos/asf/beam/blob/b8196295/sdks/python/apache_beam/io/source_test_utils.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/source_test_utils.py b/sdks/python/apache_beam/io/source_test_utils.py
index edb6409..a144a8a 100644
--- a/sdks/python/apache_beam/io/source_test_utils.py
+++ b/sdks/python/apache_beam/io/source_test_utils.py
@@ -52,6 +52,14 @@ import weakref
from multiprocessing.pool import ThreadPool
from apache_beam.io import iobase
+__all__ = ['read_from_source', 'assert_sources_equal_reference_source',
+ 'assert_reentrant_reads_succeed',
+ 'assert_split_at_fraction_behavior',
+ 'assert_split_at_fraction_binary',
+ 'assert_split_at_fraction_exhaustive',
+ 'assert_split_at_fraction_fails',
+ 'assert_split_at_fraction_succeeds_and_consistent']
+
class ExpectedSplitOutcome(object):
MUST_SUCCEED_AND_BE_CONSISTENT = 1
[2/2] beam git commit: Closes #3089
Posted by ro...@apache.org.
Closes #3089
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/0cc6dd72
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/0cc6dd72
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/0cc6dd72
Branch: refs/heads/master
Commit: 0cc6dd727c674d8724d362e261a7ca84a5ea6a57
Parents: 47cbabb b819629
Author: Robert Bradshaw <ro...@gmail.com>
Authored: Thu May 11 14:25:42 2017 -0700
Committer: Robert Bradshaw <ro...@gmail.com>
Committed: Thu May 11 14:25:42 2017 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/io/concat_source.py | 4 +++-
sdks/python/apache_beam/io/filebasedsource.py | 2 ++
sdks/python/apache_beam/io/fileio.py | 7 +++++++
sdks/python/apache_beam/io/filesystem.py | 3 +++
sdks/python/apache_beam/io/filesystems.py | 2 ++
sdks/python/apache_beam/io/gcp/gcsfilesystem.py | 2 ++
sdks/python/apache_beam/io/gcp/gcsio.py | 3 +++
sdks/python/apache_beam/io/gcp/pubsub.py | 2 ++
sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py | 3 +++
sdks/python/apache_beam/io/iobase.py | 2 ++
sdks/python/apache_beam/io/localfilesystem.py | 2 ++
sdks/python/apache_beam/io/range_trackers.py | 7 ++++++-
sdks/python/apache_beam/io/source_test_utils.py | 8 ++++++++
13 files changed, 45 insertions(+), 2 deletions(-)
----------------------------------------------------------------------