You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2016/12/06 16:41:38 UTC
[48/50] [abbrv] incubator-beam git commit: Merge remote-tracking
branch 'upstream/master' into gearpump-runner-sync
Merge remote-tracking branch 'upstream/master' into gearpump-runner-sync
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/86414c08
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/86414c08
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/86414c08
Branch: refs/heads/gearpump-runner
Commit: 86414c08eb70c8975dac7b4750ca02b8b9cf875b
Parents: 2812405 ef9871c
Author: manuzhang <ow...@gmail.com>
Authored: Mon Dec 5 11:30:31 2016 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Mon Dec 5 11:30:31 2016 +0800
----------------------------------------------------------------------
.gitignore | 4 +
.jenkins/common_job_properties.groovy | 166 ++++++
...job_beam_PostCommit_Java_MavenInstall.groovy | 42 ++
...ommit_Java_RunnableOnService_Dataflow.groovy | 39 ++
...stCommit_Java_RunnableOnService_Flink.groovy | 38 ++
...ommit_Java_RunnableOnService_Gearpump.groovy | 41 ++
...stCommit_Java_RunnableOnService_Spark.groovy | 38 ++
.../job_beam_PostCommit_Python_Verify.groovy | 37 ++
.../job_beam_PreCommit_Java_MavenInstall.groovy | 42 ++
.../job_beam_Release_NightlySnapshot.groovy | 46 ++
.jenkins/job_seed.groovy | 47 ++
.travis.yml | 2 +-
.travis/README.md | 2 +-
.../org/apache/beam/examples/complete/README.md | 14 +-
.../beam/examples/cookbook/DeDupExample.java | 96 ---
.../beam/examples/cookbook/DistinctExample.java | 96 +++
.../org/apache/beam/examples/cookbook/README.md | 16 +-
.../org/apache/beam/examples/WordCountIT.java | 2 +-
pom.xml | 10 +-
runners/apex/pom.xml | 11 +-
.../translation/ApexPipelineTranslator.java | 12 +-
.../apex/translation/TranslationContext.java | 6 +-
.../operators/ApexGroupByKeyOperator.java | 14 +-
.../core/ElementAndRestrictionCoder.java | 8 +
.../runners/core/GBKIntoKeyedWorkItems.java | 55 --
.../GroupAlsoByWindowsViaOutputBufferDoFn.java | 6 +-
.../core/GroupByKeyViaGroupByKeyOnly.java | 22 +-
.../runners/core/PerKeyCombineFnRunners.java | 263 +++++++++
.../core/PushbackSideInputDoFnRunner.java | 20 +-
.../beam/runners/core/SimpleDoFnRunner.java | 76 ++-
.../beam/runners/core/SimpleOldDoFnRunner.java | 16 -
.../beam/runners/core/SplittableParDo.java | 378 +++++++++---
.../core/GroupAlsoByWindowsProperties.java | 590 +++++++++++--------
.../core/PushbackSideInputDoFnRunnerTest.java | 18 +-
.../beam/runners/core/ReduceFnRunnerTest.java | 5 +-
.../beam/runners/core/SplittableParDoTest.java | 132 ++++-
.../beam/runners/core/WindowMatchers.java | 204 +++++++
.../beam/runners/core/WindowMatchersTest.java | 82 +++
.../direct/AbstractModelEnforcement.java | 2 +-
.../direct/BoundedReadEvaluatorFactory.java | 8 +-
.../beam/runners/direct/CommittedResult.java | 2 +-
.../beam/runners/direct/CompletionCallback.java | 2 +-
.../direct/ConsumerTrackingPipelineVisitor.java | 173 ------
...ectGBKIntoKeyedWorkItemsOverrideFactory.java | 41 +-
.../apache/beam/runners/direct/DirectGraph.java | 89 +++
.../beam/runners/direct/DirectGraphVisitor.java | 145 +++++
.../beam/runners/direct/DirectGroupByKey.java | 2 +-
.../beam/runners/direct/DirectRunner.java | 48 +-
.../runners/direct/DirectTimerInternals.java | 4 +-
.../runners/direct/DisplayDataValidator.java | 6 +-
.../runners/direct/DoFnLifecycleManager.java | 4 +-
...ecycleManagerRemovingTransformEvaluator.java | 2 +-
.../runners/direct/EmptyTransformEvaluator.java | 4 +-
.../beam/runners/direct/EvaluationContext.java | 87 ++-
.../direct/ExecutorServiceParallelExecutor.java | 17 +-
.../runners/direct/FlattenEvaluatorFactory.java | 10 +-
.../GroupAlsoByWindowEvaluatorFactory.java | 5 +-
.../direct/GroupByKeyOnlyEvaluatorFactory.java | 2 +-
.../ImmutabilityCheckingBundleFactory.java | 21 +-
.../direct/ImmutabilityEnforcementFactory.java | 2 +-
.../direct/KeyedPValueTrackingVisitor.java | 12 +-
.../beam/runners/direct/ModelEnforcement.java | 2 +-
.../beam/runners/direct/ParDoEvaluator.java | 33 +-
.../runners/direct/ParDoEvaluatorFactory.java | 94 ++-
.../direct/ParDoMultiOverrideFactory.java | 76 ++-
.../ParDoSingleViaMultiOverrideFactory.java | 6 +-
.../direct/PassthroughTransformEvaluator.java | 4 +-
...littableProcessElementsEvaluatorFactory.java | 144 +++++
.../direct/StatefulParDoEvaluatorFactory.java | 256 ++++++++
.../runners/direct/StepTransformResult.java | 38 +-
.../direct/TestStreamEvaluatorFactory.java | 2 +-
.../beam/runners/direct/TransformEvaluator.java | 2 +-
.../direct/TransformEvaluatorRegistry.java | 7 +
.../beam/runners/direct/TransformExecutor.java | 4 +-
.../beam/runners/direct/TransformResult.java | 16 +-
.../direct/UnboundedReadEvaluatorFactory.java | 3 +-
.../runners/direct/ViewEvaluatorFactory.java | 2 +-
.../direct/WatermarkCallbackExecutor.java | 34 ++
.../beam/runners/direct/WatermarkManager.java | 50 +-
.../runners/direct/WindowEvaluatorFactory.java | 6 +-
.../direct/BoundedReadEvaluatorFactoryTest.java | 16 +-
.../ConsumerTrackingPipelineVisitorTest.java | 287 ---------
.../runners/direct/DirectGraphVisitorTest.java | 239 ++++++++
...leManagerRemovingTransformEvaluatorTest.java | 4 +-
.../runners/direct/EvaluationContextTest.java | 49 +-
.../direct/FlattenEvaluatorFactoryTest.java | 6 +-
.../ImmutabilityCheckingBundleFactoryTest.java | 6 +-
.../ImmutabilityEnforcementFactoryTest.java | 6 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 2 +-
.../beam/runners/direct/SplittableDoFnTest.java | 231 --------
.../StatefulParDoEvaluatorFactoryTest.java | 300 ++++++++++
.../runners/direct/StepTransformResultTest.java | 25 +-
.../direct/TestStreamEvaluatorFactoryTest.java | 10 +-
.../runners/direct/TransformExecutorTest.java | 223 +++----
.../UnboundedReadEvaluatorFactoryTest.java | 20 +-
.../runners/direct/WatermarkManagerTest.java | 23 +-
.../direct/WindowEvaluatorFactoryTest.java | 16 +-
runners/flink/runner/pom.xml | 9 +-
.../apache/beam/runners/flink/FlinkRunner.java | 12 +-
.../FlinkBatchPipelineTranslator.java | 14 +-
.../FlinkStreamingPipelineTranslator.java | 16 +-
.../FlinkStreamingTransformTranslators.java | 63 +-
.../PipelineTranslationOptimizer.java | 10 +-
.../FlinkMergingNonShuffleReduceFunction.java | 2 +-
.../FlinkMergingPartialReduceFunction.java | 2 +-
.../functions/FlinkMergingReduceFunction.java | 2 +-
.../functions/FlinkPartialReduceFunction.java | 2 +-
.../functions/FlinkProcessContextBase.java | 8 -
.../functions/FlinkReduceFunction.java | 2 +-
runners/google-cloud-dataflow-java/pom.xml | 4 +
.../dataflow/DataflowPipelineTranslator.java | 33 +-
.../beam/runners/dataflow/DataflowRunner.java | 53 +-
.../DataflowPipelineTranslatorTest.java | 2 +-
.../runners/dataflow/DataflowRunnerTest.java | 4 +-
.../dataflow/RecordingPipelineVisitor.java | 6 +-
runners/spark/pom.xml | 7 +-
.../runners/spark/SparkPipelineOptions.java | 5 +
.../apache/beam/runners/spark/SparkRunner.java | 80 ++-
.../spark/aggregators/AccumulatorSingleton.java | 2 +-
.../beam/runners/spark/io/MicrobatchSource.java | 20 +-
.../beam/runners/spark/io/SourceDStream.java | 3 +-
.../spark/stateful/StateSpecFunctions.java | 2 +-
.../spark/translation/BoundedDataset.java | 5 +-
.../beam/runners/spark/translation/Dataset.java | 2 +-
.../spark/translation/EvaluationContext.java | 10 +-
.../spark/translation/SparkProcessContext.java | 9 -
.../translation/StorageLevelPTransform.java | 43 ++
.../spark/translation/TransformTranslator.java | 27 +
.../SparkRunnerStreamingContextFactory.java | 2 +-
.../streaming/StreamingTransformTranslator.java | 2 +-
.../translation/streaming/UnboundedDataset.java | 13 +-
.../spark/translation/StorageLevelTest.java | 56 ++
.../streaming/EmptyStreamAssertionTest.java | 2 +
.../streaming/FlattenStreamingTest.java | 2 +
.../streaming/SimpleStreamingWordCountTest.java | 1 +
.../SparkTestPipelineOptionsForStreaming.java | 6 -
sdks/java/core/pom.xml | 29 +-
.../beam/sdk/AggregatorPipelineExtractor.java | 6 +-
.../main/java/org/apache/beam/sdk/Pipeline.java | 151 ++---
.../org/apache/beam/sdk/io/FileBasedSink.java | 22 +-
.../java/org/apache/beam/sdk/io/PubsubIO.java | 176 +++++-
.../apache/beam/sdk/io/PubsubUnboundedSink.java | 23 +-
.../beam/sdk/io/PubsubUnboundedSource.java | 40 +-
.../java/org/apache/beam/sdk/io/TextIO.java | 28 +-
.../java/org/apache/beam/sdk/io/XmlSink.java | 4 +-
.../beam/sdk/options/PipelineOptions.java | 7 +
.../sdk/options/PipelineOptionsFactory.java | 1 +
.../sdk/options/ProxyInvocationHandler.java | 26 +
.../apache/beam/sdk/options/ValueProvider.java | 2 +-
.../beam/sdk/runners/TransformHierarchy.java | 340 +++++++++--
.../beam/sdk/runners/TransformTreeNode.java | 257 --------
.../beam/sdk/testing/FileChecksumMatcher.java | 168 +++++-
.../apache/beam/sdk/testing/GatherAllPanes.java | 88 +++
.../org/apache/beam/sdk/testing/PAssert.java | 77 +--
.../apache/beam/sdk/testing/PaneExtractors.java | 55 +-
.../beam/sdk/testing/UsesSplittableParDo.java | 25 +
.../beam/sdk/testing/ValueInSingleWindow.java | 134 +++++
.../org/apache/beam/sdk/transforms/Combine.java | 37 +-
.../org/apache/beam/sdk/transforms/DoFn.java | 16 +-
.../beam/sdk/transforms/DoFnAdapters.java | 2 +
.../apache/beam/sdk/transforms/DoFnTester.java | 350 ++++++-----
.../org/apache/beam/sdk/transforms/OldDoFn.java | 8 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 10 -
.../reflect/ByteBuddyDoFnInvokerFactory.java | 28 +-
.../reflect/ByteBuddyOnTimerInvokerFactory.java | 10 +-
.../sdk/transforms/reflect/DoFnInvoker.java | 3 +
.../sdk/transforms/reflect/DoFnInvokers.java | 16 +-
.../sdk/transforms/reflect/DoFnSignature.java | 8 +
.../apache/beam/sdk/util/GatherAllPanes.java | 86 ---
.../apache/beam/sdk/util/IdentityWindowFn.java | 2 +-
.../beam/sdk/util/PerKeyCombineFnRunners.java | 258 --------
.../apache/beam/sdk/util/TimerInternals.java | 107 ++--
.../beam/sdk/util/WindowingInternals.java | 10 -
.../beam/sdk/util/common/ReflectHelpers.java | 3 +-
.../sdk/util/state/InMemoryTimerInternals.java | 8 +-
.../sdk/util/state/TimerInternalsFactory.java | 36 ++
.../sdk/AggregatorPipelineExtractorTest.java | 20 +-
.../org/apache/beam/sdk/WindowMatchers.java | 204 -------
.../org/apache/beam/sdk/WindowMatchersTest.java | 82 ---
.../org/apache/beam/sdk/io/PubsubIOTest.java | 43 +-
.../beam/sdk/io/PubsubUnboundedSinkTest.java | 20 +-
.../beam/sdk/io/PubsubUnboundedSourceTest.java | 14 +-
.../org/apache/beam/sdk/io/XmlSinkTest.java | 6 +-
.../beam/sdk/options/PipelineOptionsTest.java | 24 +
.../beam/sdk/options/ValueProviderTest.java | 18 +
.../sdk/runners/TransformHierarchyTest.java | 230 ++++++++
.../beam/sdk/runners/TransformTreeTest.java | 12 +-
.../sdk/testing/FileChecksumMatcherTest.java | 131 +++-
.../beam/sdk/testing/GatherAllPanesTest.java | 140 +++++
.../beam/sdk/testing/PaneExtractorsTest.java | 133 ++---
.../testing/ValueInSingleWindowCoderTest.java | 51 ++
.../beam/sdk/transforms/DoFnTesterTest.java | 38 +-
.../apache/beam/sdk/transforms/ParDoTest.java | 177 +++++-
.../beam/sdk/transforms/SplittableDoFnTest.java | 401 +++++++++++++
.../display/DisplayDataEvaluator.java | 8 +-
.../beam/sdk/util/GatherAllPanesTest.java | 143 -----
.../apache/beam/sdk/util/ReleaseInfoTest.java | 45 ++
.../beam/sdk/util/TimerInternalsTest.java | 4 +-
.../sorter/BufferedExternalSorter.java | 6 +-
.../sdk/extensions/sorter/ExternalSorter.java | 6 +-
.../sorter/BufferedExternalSorterTest.java | 58 +-
.../extensions/sorter/ExternalSorterTest.java | 53 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 52 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 26 +-
.../apache/beam/sdk/io/mongodb/MongoDbIO.java | 6 +-
.../examples-java8/generate-sources.sh | 82 +++
.../maven-archetypes/examples-java8/pom.xml | 177 ++++++
.../META-INF/maven/archetype-metadata.xml | 39 ++
.../main/resources/archetype-resources/pom.xml | 209 +++++++
.../projects/basic/archetype.properties | 21 +
.../src/test/resources/projects/basic/goal.txt | 1 +
.../examples/generate-sources.sh | 68 +++
sdks/java/maven-archetypes/examples/pom.xml | 27 +-
.../main/resources/archetype-resources/pom.xml | 123 +++-
.../src/main/java/DebuggingWordCount.java | 164 ------
.../src/main/java/MinimalWordCount.java | 118 ----
.../src/main/java/WindowedWordCount.java | 229 -------
.../src/main/java/WordCount.java | 186 ------
.../common/ExampleBigQueryTableOptions.java | 55 --
.../src/main/java/common/ExampleOptions.java | 37 --
...xamplePubsubTopicAndSubscriptionOptions.java | 45 --
.../java/common/ExamplePubsubTopicOptions.java | 45 --
.../src/main/java/common/ExampleUtils.java | 352 -----------
.../src/test/java/DebuggingWordCountTest.java | 52 --
.../src/test/java/WordCountTest.java | 85 ---
sdks/java/maven-archetypes/pom.xml | 38 +-
.../main/resources/archetype-resources/pom.xml | 8 +
.../resources/projects/basic/reference/pom.xml | 8 +
.../update-examples-archetype.sh | 59 --
229 files changed, 7944 insertions(+), 5261 deletions(-)
----------------------------------------------------------------------