You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by jk...@apache.org on 2017/07/11 01:24:08 UTC
[5/5] beam git commit: This closes #3356: [BEAM-92] Allow
value-dependent files in FileBasedSink
This closes #3356: [BEAM-92] Allow value-dependent files in FileBasedSink
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/c14a3184
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/c14a3184
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/c14a3184
Branch: refs/heads/master
Commit: c14a3184e69f3ba6d228a61b2218930537008da8
Parents: 9d48bd5 77ba7a3
Author: Eugene Kirpichov <ki...@google.com>
Authored: Mon Jul 10 18:10:19 2017 -0700
Committer: Eugene Kirpichov <ki...@google.com>
Committed: Mon Jul 10 18:10:19 2017 -0700
----------------------------------------------------------------------
.../examples/common/WriteOneFilePerWindow.java | 52 +-
.../beam/examples/WindowedWordCountIT.java | 4 +-
.../complete/game/utils/WriteToText.java | 43 +-
.../construction/WriteFilesTranslation.java | 67 +-
.../construction/PTransformMatchersTest.java | 22 +-
.../construction/WriteFilesTranslationTest.java | 62 +-
.../direct/WriteWithShardingFactory.java | 6 +-
.../direct/WriteWithShardingFactoryTest.java | 18 +-
.../beam/runners/dataflow/DataflowRunner.java | 15 +-
.../runners/dataflow/DataflowRunnerTest.java | 35 +-
.../runners/spark/SparkRunnerDebuggerTest.java | 26 +-
.../src/main/proto/beam_runner_api.proto | 7 +-
.../apache/beam/sdk/coders/ShardedKeyCoder.java | 66 ++
.../java/org/apache/beam/sdk/io/AvroIO.java | 220 ++++---
.../java/org/apache/beam/sdk/io/AvroSink.java | 32 +-
.../beam/sdk/io/DefaultFilenamePolicy.java | 274 +++++---
.../beam/sdk/io/DynamicFileDestinations.java | 115 ++++
.../org/apache/beam/sdk/io/FileBasedSink.java | 513 ++++++++-------
.../java/org/apache/beam/sdk/io/TFRecordIO.java | 44 +-
.../java/org/apache/beam/sdk/io/TextIO.java | 488 ++++++++++----
.../java/org/apache/beam/sdk/io/TextSink.java | 22 +-
.../java/org/apache/beam/sdk/io/WriteFiles.java | 640 +++++++++++--------
.../sdk/transforms/SerializableFunctions.java | 50 ++
.../org/apache/beam/sdk/values/ShardedKey.java | 65 ++
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 85 ++-
.../beam/sdk/io/DefaultFilenamePolicyTest.java | 135 ++--
.../sdk/io/DrunkWritableByteChannelFactory.java | 2 +-
.../apache/beam/sdk/io/FileBasedSinkTest.java | 93 +--
.../java/org/apache/beam/sdk/io/SimpleSink.java | 56 +-
.../java/org/apache/beam/sdk/io/TextIOTest.java | 264 +++++++-
.../org/apache/beam/sdk/io/WriteFilesTest.java | 339 ++++++++--
.../beam/sdk/io/gcp/bigquery/BatchLoads.java | 2 +
.../io/gcp/bigquery/DynamicDestinations.java | 29 +-
.../io/gcp/bigquery/GenerateShardedTable.java | 1 +
.../beam/sdk/io/gcp/bigquery/ShardedKey.java | 67 --
.../sdk/io/gcp/bigquery/ShardedKeyCoder.java | 74 ---
.../sdk/io/gcp/bigquery/StreamingWriteFn.java | 1 +
.../io/gcp/bigquery/StreamingWriteTables.java | 2 +
.../sdk/io/gcp/bigquery/TagWithUniqueIds.java | 1 +
.../io/gcp/bigquery/WriteBundlesToFiles.java | 2 +
.../bigquery/WriteGroupedRecordsToFiles.java | 1 +
.../sdk/io/gcp/bigquery/WritePartition.java | 1 +
.../beam/sdk/io/gcp/bigquery/WriteTables.java | 1 +
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 2 +
.../java/org/apache/beam/sdk/io/xml/XmlIO.java | 4 +-
.../org/apache/beam/sdk/io/xml/XmlSink.java | 21 +-
.../org/apache/beam/sdk/io/xml/XmlSinkTest.java | 4 +-
47 files changed, 2710 insertions(+), 1363 deletions(-)
----------------------------------------------------------------------