You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by cc...@apache.org on 2018/07/06 21:18:11 UTC
[beam] branch master updated (3430d68 -> 6f6feaa)
This is an automated email from the ASF dual-hosted git repository.
ccy pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git.
from 3430d68 Merge pull request #5886 from Split bounded sources in larger than 1k chunks.
add 8bf17d4 Futurize io subpackage
add 071bc35 incorporated all feedback for futurize io subpackage
new 6f6feaa Merge pull request #5715 from Fematich/io
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
sdks/python/apache_beam/io/__init__.py | 2 ++
sdks/python/apache_beam/io/avroio.py | 8 +++--
sdks/python/apache_beam/io/avroio_test.py | 2 ++
sdks/python/apache_beam/io/concat_source.py | 3 ++
sdks/python/apache_beam/io/concat_source_test.py | 22 +++++++-----
sdks/python/apache_beam/io/filebasedsink.py | 21 ++++++++----
sdks/python/apache_beam/io/filebasedsink_test.py | 3 ++
sdks/python/apache_beam/io/filebasedsource.py | 14 ++++----
sdks/python/apache_beam/io/filebasedsource_test.py | 29 +++++++++-------
sdks/python/apache_beam/io/filesystem.py | 37 +++++++++++++-------
sdks/python/apache_beam/io/filesystem_test.py | 40 ++++++++++++++++------
sdks/python/apache_beam/io/filesystemio.py | 13 +++----
sdks/python/apache_beam/io/filesystemio_test.py | 3 ++
sdks/python/apache_beam/io/filesystems.py | 13 +++++--
sdks/python/apache_beam/io/filesystems_test.py | 12 ++++---
sdks/python/apache_beam/io/gcp/__init__.py | 1 +
sdks/python/apache_beam/io/gcp/bigquery.py | 17 ++++++---
sdks/python/apache_beam/io/gcp/bigquery_test.py | 4 ++-
.../apache_beam/io/gcp/datastore/__init__.py | 1 +
.../apache_beam/io/gcp/datastore/v1/__init__.py | 1 +
.../io/gcp/datastore/v1/adaptive_throttler.py | 4 +++
.../io/gcp/datastore/v1/adaptive_throttler_test.py | 4 +++
.../apache_beam/io/gcp/datastore/v1/datastoreio.py | 9 +++--
.../io/gcp/datastore/v1/datastoreio_test.py | 10 ++++--
.../io/gcp/datastore/v1/fake_datastore.py | 3 ++
.../apache_beam/io/gcp/datastore/v1/helper.py | 12 ++++---
.../apache_beam/io/gcp/datastore/v1/helper_test.py | 3 ++
.../io/gcp/datastore/v1/query_splitter.py | 4 +++
.../io/gcp/datastore/v1/query_splitter_test.py | 2 ++
.../python/apache_beam/io/gcp/datastore/v1/util.py | 5 +++
sdks/python/apache_beam/io/gcp/gcsfilesystem.py | 6 +++-
.../apache_beam/io/gcp/gcsfilesystem_test.py | 1 +
sdks/python/apache_beam/io/gcp/gcsio.py | 7 ++--
sdks/python/apache_beam/io/gcp/gcsio_test.py | 7 ++--
.../python/apache_beam/io/gcp/internal/__init__.py | 1 +
.../io/gcp/internal/clients/__init__.py | 1 +
.../io/gcp/internal/clients/bigquery/__init__.py | 2 ++
.../clients/bigquery/bigquery_v2_client.py | 5 ++-
.../clients/bigquery/bigquery_v2_messages.py | 4 +--
.../io/gcp/internal/clients/storage/__init__.py | 2 ++
.../internal/clients/storage/storage_v1_client.py | 6 +++-
.../clients/storage/storage_v1_messages.py | 5 +--
sdks/python/apache_beam/io/gcp/pubsub.py | 6 ++--
sdks/python/apache_beam/io/gcp/pubsub_test.py | 5 +++
sdks/python/apache_beam/io/gcp/tests/__init__.py | 2 ++
.../apache_beam/io/gcp/tests/bigquery_matcher.py | 2 ++
.../io/gcp/tests/bigquery_matcher_test.py | 2 ++
.../apache_beam/io/gcp/tests/pubsub_matcher.py | 2 ++
sdks/python/apache_beam/io/gcp/tests/utils.py | 2 ++
sdks/python/apache_beam/io/gcp/tests/utils_test.py | 2 ++
sdks/python/apache_beam/io/hadoopfilesystem.py | 1 +
.../python/apache_beam/io/hadoopfilesystem_test.py | 5 ++-
sdks/python/apache_beam/io/iobase.py | 4 +++
sdks/python/apache_beam/io/localfilesystem.py | 1 +
sdks/python/apache_beam/io/localfilesystem_test.py | 12 ++++---
sdks/python/apache_beam/io/range_trackers.py | 15 +++++---
sdks/python/apache_beam/io/range_trackers_test.py | 13 ++++---
sdks/python/apache_beam/io/restriction_trackers.py | 5 ++-
.../apache_beam/io/restriction_trackers_test.py | 2 ++
sdks/python/apache_beam/io/source_test_utils.py | 5 +++
.../apache_beam/io/source_test_utils_test.py | 3 ++
sdks/python/apache_beam/io/sources_test.py | 1 +
sdks/python/apache_beam/io/textio.py | 2 ++
sdks/python/apache_beam/io/textio_test.py | 3 ++
sdks/python/apache_beam/io/tfrecordio.py | 1 +
sdks/python/apache_beam/io/tfrecordio_test.py | 26 ++++++++------
sdks/python/apache_beam/io/utils.py | 4 +++
sdks/python/apache_beam/io/vcfio.py | 15 +++++---
sdks/python/apache_beam/io/vcfio_test.py | 9 +++--
69 files changed, 362 insertions(+), 137 deletions(-)
[beam] 01/01: Merge pull request #5715 from Fematich/io
Posted by cc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ccy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 6f6feaaeebfc82302ba83c52d087b06a12a5b119
Merge: 3430d68 071bc35
Author: Charles Chen <ch...@users.noreply.github.com>
AuthorDate: Fri Jul 6 14:18:06 2018 -0700
Merge pull request #5715 from Fematich/io
[BEAM-4000] Futurize io subpackage
sdks/python/apache_beam/io/__init__.py | 2 ++
sdks/python/apache_beam/io/avroio.py | 8 +++--
sdks/python/apache_beam/io/avroio_test.py | 2 ++
sdks/python/apache_beam/io/concat_source.py | 3 ++
sdks/python/apache_beam/io/concat_source_test.py | 22 +++++++-----
sdks/python/apache_beam/io/filebasedsink.py | 21 ++++++++----
sdks/python/apache_beam/io/filebasedsink_test.py | 3 ++
sdks/python/apache_beam/io/filebasedsource.py | 14 ++++----
sdks/python/apache_beam/io/filebasedsource_test.py | 29 +++++++++-------
sdks/python/apache_beam/io/filesystem.py | 37 +++++++++++++-------
sdks/python/apache_beam/io/filesystem_test.py | 40 ++++++++++++++++------
sdks/python/apache_beam/io/filesystemio.py | 13 +++----
sdks/python/apache_beam/io/filesystemio_test.py | 3 ++
sdks/python/apache_beam/io/filesystems.py | 13 +++++--
sdks/python/apache_beam/io/filesystems_test.py | 12 ++++---
sdks/python/apache_beam/io/gcp/__init__.py | 1 +
sdks/python/apache_beam/io/gcp/bigquery.py | 17 ++++++---
sdks/python/apache_beam/io/gcp/bigquery_test.py | 4 ++-
.../apache_beam/io/gcp/datastore/__init__.py | 1 +
.../apache_beam/io/gcp/datastore/v1/__init__.py | 1 +
.../io/gcp/datastore/v1/adaptive_throttler.py | 4 +++
.../io/gcp/datastore/v1/adaptive_throttler_test.py | 4 +++
.../apache_beam/io/gcp/datastore/v1/datastoreio.py | 9 +++--
.../io/gcp/datastore/v1/datastoreio_test.py | 10 ++++--
.../io/gcp/datastore/v1/fake_datastore.py | 3 ++
.../apache_beam/io/gcp/datastore/v1/helper.py | 12 ++++---
.../apache_beam/io/gcp/datastore/v1/helper_test.py | 3 ++
.../io/gcp/datastore/v1/query_splitter.py | 4 +++
.../io/gcp/datastore/v1/query_splitter_test.py | 2 ++
.../python/apache_beam/io/gcp/datastore/v1/util.py | 5 +++
sdks/python/apache_beam/io/gcp/gcsfilesystem.py | 6 +++-
.../apache_beam/io/gcp/gcsfilesystem_test.py | 1 +
sdks/python/apache_beam/io/gcp/gcsio.py | 7 ++--
sdks/python/apache_beam/io/gcp/gcsio_test.py | 7 ++--
.../python/apache_beam/io/gcp/internal/__init__.py | 1 +
.../io/gcp/internal/clients/__init__.py | 1 +
.../io/gcp/internal/clients/bigquery/__init__.py | 2 ++
.../clients/bigquery/bigquery_v2_client.py | 5 ++-
.../clients/bigquery/bigquery_v2_messages.py | 4 +--
.../io/gcp/internal/clients/storage/__init__.py | 2 ++
.../internal/clients/storage/storage_v1_client.py | 6 +++-
.../clients/storage/storage_v1_messages.py | 5 +--
sdks/python/apache_beam/io/gcp/pubsub.py | 6 ++--
sdks/python/apache_beam/io/gcp/pubsub_test.py | 5 +++
sdks/python/apache_beam/io/gcp/tests/__init__.py | 2 ++
.../apache_beam/io/gcp/tests/bigquery_matcher.py | 2 ++
.../io/gcp/tests/bigquery_matcher_test.py | 2 ++
.../apache_beam/io/gcp/tests/pubsub_matcher.py | 2 ++
sdks/python/apache_beam/io/gcp/tests/utils.py | 2 ++
sdks/python/apache_beam/io/gcp/tests/utils_test.py | 2 ++
sdks/python/apache_beam/io/hadoopfilesystem.py | 1 +
.../python/apache_beam/io/hadoopfilesystem_test.py | 5 ++-
sdks/python/apache_beam/io/iobase.py | 4 +++
sdks/python/apache_beam/io/localfilesystem.py | 1 +
sdks/python/apache_beam/io/localfilesystem_test.py | 12 ++++---
sdks/python/apache_beam/io/range_trackers.py | 15 +++++---
sdks/python/apache_beam/io/range_trackers_test.py | 13 ++++---
sdks/python/apache_beam/io/restriction_trackers.py | 5 ++-
.../apache_beam/io/restriction_trackers_test.py | 2 ++
sdks/python/apache_beam/io/source_test_utils.py | 5 +++
.../apache_beam/io/source_test_utils_test.py | 3 ++
sdks/python/apache_beam/io/sources_test.py | 1 +
sdks/python/apache_beam/io/textio.py | 2 ++
sdks/python/apache_beam/io/textio_test.py | 3 ++
sdks/python/apache_beam/io/tfrecordio.py | 1 +
sdks/python/apache_beam/io/tfrecordio_test.py | 26 ++++++++------
sdks/python/apache_beam/io/utils.py | 4 +++
sdks/python/apache_beam/io/vcfio.py | 15 +++++---
sdks/python/apache_beam/io/vcfio_test.py | 9 +++--
69 files changed, 362 insertions(+), 137 deletions(-)
diff --cc sdks/python/apache_beam/io/iobase.py
index 3629e8f,34a50b5..459c326
--- a/sdks/python/apache_beam/io/iobase.py
+++ b/sdks/python/apache_beam/io/iobase.py
@@@ -29,10 -29,13 +29,14 @@@ returns a writer object supporting writ
the sink.
"""
+ from __future__ import absolute_import
+
import logging
+import math
import random
import uuid
+ from builtins import object
+ from builtins import range
from collections import namedtuple
from apache_beam import coders