You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ro...@apache.org on 2016/10/27 17:28:23 UTC
[35/50] [abbrv] incubator-beam git commit: This closes #1143
This closes #1143
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/dc61a001
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/dc61a001
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/dc61a001
Branch: refs/heads/python-sdk
Commit: dc61a001c1b9389450ade3f49b98a9a22c10f794
Parents: c94b8ea b72e7e3
Author: Sela <an...@paypal.com>
Authored: Wed Oct 26 20:52:17 2016 +0300
Committer: Sela <an...@paypal.com>
Committed: Wed Oct 26 20:52:17 2016 +0300
----------------------------------------------------------------------
runners/spark/pom.xml | 44 ++--
.../runners/spark/SparkPipelineOptions.java | 11 +
.../runners/spark/io/EmptyCheckpointMark.java | 52 ++++
.../apache/beam/runners/spark/io/KafkaIO.java | 131 ----------
.../beam/runners/spark/io/MicrobatchSource.java | 262 +++++++++++++++++++
.../beam/runners/spark/io/SourceDStream.java | 156 +++++++++++
.../apache/beam/runners/spark/io/SourceRDD.java | 75 +++++-
.../runners/spark/io/SparkUnboundedSource.java | 167 ++++++++++++
.../spark/stateful/StateSpecFunctions.java | 167 ++++++++++++
.../runners/spark/stateful/package-info.java | 22 ++
.../spark/translation/TranslationUtils.java | 17 ++
.../SparkRunnerStreamingContextFactory.java | 7 +
.../streaming/StreamingTransformTranslator.java | 37 +--
.../streaming/KafkaStreamingTest.java | 150 ++++++++---
.../ResumeFromCheckpointStreamingTest.java | 34 ++-
.../streaming/utils/EmbeddedKafkaCluster.java | 51 ++--
.../utils/KafkaWriteOnBatchCompleted.java | 105 ++++++++
17 files changed, 1224 insertions(+), 264 deletions(-)
----------------------------------------------------------------------