You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by am...@apache.org on 2016/09/21 17:25:09 UTC
[4/4] incubator-beam git commit: This closes #909
This closes #909
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/1ceb12ae
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/1ceb12ae
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/1ceb12ae
Branch: refs/heads/master
Commit: 1ceb12aebd0ffa63bd28d31cbe830230713705ec
Parents: 5c23f49 0feb649
Author: Sela <an...@paypal.com>
Authored: Wed Sep 21 20:17:38 2016 +0300
Committer: Sela <an...@paypal.com>
Committed: Wed Sep 21 20:17:38 2016 +0300
----------------------------------------------------------------------
.../runners/spark/SparkPipelineOptions.java | 28 +-
.../apache/beam/runners/spark/SparkRunner.java | 121 ++--
.../spark/aggregators/AccumulatorSingleton.java | 53 ++
.../runners/spark/translation/DoFnFunction.java | 35 +-
.../spark/translation/EvaluationContext.java | 17 +-
.../translation/GroupCombineFunctions.java | 262 +++++++++
.../spark/translation/MultiDoFnFunction.java | 44 +-
.../spark/translation/SparkContextFactory.java | 48 +-
.../translation/SparkPipelineEvaluator.java | 57 --
.../translation/SparkPipelineTranslator.java | 5 +-
.../spark/translation/SparkProcessContext.java | 10 +-
.../spark/translation/SparkRuntimeContext.java | 44 +-
.../spark/translation/TransformTranslator.java | 473 +++-------------
.../spark/translation/TranslationUtils.java | 195 +++++++
.../SparkRunnerStreamingContextFactory.java | 98 ++++
.../streaming/StreamingEvaluationContext.java | 44 +-
.../streaming/StreamingTransformTranslator.java | 549 ++++++++++++-------
.../runners/spark/util/BroadcastHelper.java | 26 +
.../runners/spark/ClearAggregatorsRule.java | 33 ++
.../beam/runners/spark/SimpleWordCountTest.java | 4 +
.../spark/translation/SideEffectsTest.java | 3 +-
.../streaming/FlattenStreamingTest.java | 54 +-
.../streaming/KafkaStreamingTest.java | 26 +-
.../RecoverFromCheckpointStreamingTest.java | 179 ++++++
.../streaming/SimpleStreamingWordCountTest.java | 25 +-
.../utils/TestOptionsForStreaming.java | 55 ++
.../org/apache/beam/sdk/transforms/Combine.java | 7 +
27 files changed, 1682 insertions(+), 813 deletions(-)
----------------------------------------------------------------------