You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2017/07/24 04:35:10 UTC
[27/50] [abbrv] beam git commit: Fix documentation for the
shard_template_name
Fix documentation for the shard_template_name
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/b3f74044
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/b3f74044
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/b3f74044
Branch: refs/heads/jstorm-runner
Commit: b3f74044436f4b4a92f3135411b5d5093437ec3e
Parents: 13e1be2
Author: Sourabh Bajaj <so...@google.com>
Authored: Mon May 15 14:34:45 2017 -0700
Committer: Sourabh Bajaj <so...@google.com>
Committed: Mon May 15 14:34:45 2017 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/io/avroio.py | 13 ++++++-------
sdks/python/apache_beam/io/textio.py | 13 ++++++-------
sdks/python/apache_beam/io/tfrecordio.py | 15 +++++++--------
3 files changed, 19 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/b3f74044/sdks/python/apache_beam/io/avroio.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/avroio.py b/sdks/python/apache_beam/io/avroio.py
index e02e1f7..7df9983 100644
--- a/sdks/python/apache_beam/io/avroio.py
+++ b/sdks/python/apache_beam/io/avroio.py
@@ -312,13 +312,12 @@ class WriteToAvro(beam.transforms.PTransform):
the performance of a pipeline. Setting this value is not recommended
unless you require a specific number of output files.
shard_name_template: A template string containing placeholders for
- the shard number and shard count. Currently only '' and
- '-SSSSS-of-NNNNN' are patterns accepted by the service.
- When constructing a filename for a particular shard number, the
- upper-case letters 'S' and 'N' are replaced with the 0-padded shard
- number and shard count respectively. This argument can be '' in which
- case it behaves as if num_shards was set to 1 and only one file will be
- generated. The default pattern used is '-SSSSS-of-NNNNN'.
+ the shard number and shard count. When constructing a filename for a
+ particular shard number, the upper-case letters 'S' and 'N' are
+ replaced with the 0-padded shard number and shard count respectively.
+ This argument can be '' in which case it behaves as if num_shards was
+ set to 1 and only one file will be generated. The default pattern used
+ is '-SSSSS-of-NNNNN' if None is passed as the shard_name_template.
mime_type: The MIME type to use for the produced files, if the filesystem
supports specifying MIME types.
http://git-wip-us.apache.org/repos/asf/beam/blob/b3f74044/sdks/python/apache_beam/io/textio.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/textio.py b/sdks/python/apache_beam/io/textio.py
index eeefaf6..60e1512 100644
--- a/sdks/python/apache_beam/io/textio.py
+++ b/sdks/python/apache_beam/io/textio.py
@@ -291,13 +291,12 @@ class _TextSink(filebasedsink.FileBasedSink):
the performance of a pipeline. Setting this value is not recommended
unless you require a specific number of output files.
shard_name_template: A template string containing placeholders for
- the shard number and shard count. Currently only '' and
- '-SSSSS-of-NNNNN' are patterns accepted by the service.
- When constructing a filename for a particular shard number, the
- upper-case letters 'S' and 'N' are replaced with the 0-padded shard
- number and shard count respectively. This argument can be '' in which
- case it behaves as if num_shards was set to 1 and only one file will be
- generated. The default pattern used is '-SSSSS-of-NNNNN'.
+ the shard number and shard count. When constructing a filename for a
+ particular shard number, the upper-case letters 'S' and 'N' are
+ replaced with the 0-padded shard number and shard count respectively.
+ This argument can be '' in which case it behaves as if num_shards was
+ set to 1 and only one file will be generated. The default pattern used
+ is '-SSSSS-of-NNNNN' if None is passed as the shard_name_template.
coder: Coder used to encode each line.
compression_type: Used to handle compressed output files. Typical value
is CompressionTypes.AUTO, in which case the final file path's
http://git-wip-us.apache.org/repos/asf/beam/blob/b3f74044/sdks/python/apache_beam/io/tfrecordio.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/tfrecordio.py b/sdks/python/apache_beam/io/tfrecordio.py
index a8cd1ce..d7eb932 100644
--- a/sdks/python/apache_beam/io/tfrecordio.py
+++ b/sdks/python/apache_beam/io/tfrecordio.py
@@ -242,7 +242,7 @@ class WriteToTFRecord(PTransform):
coder=coders.BytesCoder(),
file_name_suffix='',
num_shards=0,
- shard_name_template=filebasedsink.DEFAULT_SHARD_NAME_TEMPLATE,
+ shard_name_template=None,
compression_type=CompressionTypes.AUTO,
**kwargs):
"""Initialize WriteToTFRecord transform.
@@ -256,13 +256,12 @@ class WriteToTFRecord(PTransform):
num_shards: The number of files (shards) used for output. If not set, the
default value will be used.
shard_name_template: A template string containing placeholders for
- the shard number and shard count. Currently only '' and
- '-SSSSS-of-NNNNN' are patterns allowed.
- When constructing a filename for a particular shard number, the
- upper-case letters 'S' and 'N' are replaced with the 0-padded shard
- number and shard count respectively. This argument can be '' in which
- case it behaves as if num_shards was set to 1 and only one file will be
- generated. The default pattern is '-SSSSS-of-NNNNN'.
+ the shard number and shard count. When constructing a filename for a
+ particular shard number, the upper-case letters 'S' and 'N' are
+ replaced with the 0-padded shard number and shard count respectively.
+ This argument can be '' in which case it behaves as if num_shards was
+ set to 1 and only one file will be generated. The default pattern used
+ is '-SSSSS-of-NNNNN' if None is passed as the shard_name_template.
compression_type: Used to handle compressed output files. Typical value
is CompressionTypes.AUTO, in which case the file_path's extension will
be used to detect the compression.