You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2016/11/23 06:52:46 UTC
[50/50] incubator-beam git commit: Merge branch 'master' of
https://github.com/apache/incubator-beam into gearpump-runner
Merge branch 'master' of https://github.com/apache/incubator-beam into gearpump-runner
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/28124050
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/28124050
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/28124050
Branch: refs/heads/gearpump-runner
Commit: 28124050d5598acf8c9df621551ed2d84dc643fd
Parents: 68363d0 dd85cad
Author: manuzhang <ow...@gmail.com>
Authored: Wed Nov 23 12:32:37 2016 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Wed Nov 23 12:32:37 2016 +0800
----------------------------------------------------------------------
README.md | 4 +-
pom.xml | 34 +-
runners/apex/pom.xml | 3 +-
.../translation/ParDoBoundMultiTranslator.java | 67 ++-
.../apex/translation/ParDoBoundTranslator.java | 46 +-
.../operators/ApexGroupByKeyOperator.java | 26 +-
runners/core-java/pom.xml | 2 +-
.../core/GroupAlsoByWindowViaWindowSetDoFn.java | 3 +-
.../GroupAlsoByWindowsViaOutputBufferDoFn.java | 5 +-
.../beam/runners/core/OutputWindowedValue.java | 45 ++
.../beam/runners/core/PaneInfoTracker.java | 3 -
.../runners/core/ReduceFnContextFactory.java | 83 +++-
.../beam/runners/core/ReduceFnRunner.java | 40 +-
.../beam/runners/core/SimpleDoFnRunner.java | 142 ++++--
.../beam/runners/core/SimpleOldDoFnRunner.java | 21 +-
.../beam/runners/core/SplittableParDo.java | 35 +-
.../beam/runners/core/SystemReduceFn.java | 3 -
.../apache/beam/runners/core/WatermarkHold.java | 3 -
.../core/WindowingInternalsAdapters.java | 76 ++++
.../AfterDelayFromFirstElementStateMachine.java | 5 -
.../core/triggers/AfterPaneStateMachine.java | 3 -
.../core/triggers/TriggerStateMachines.java | 15 -
.../beam/runners/core/ReduceFnTester.java | 81 +---
runners/direct-java/pom.xml | 2 +-
.../direct/BoundedReadEvaluatorFactory.java | 110 ++++-
.../beam/runners/direct/DirectRunner.java | 18 +-
.../beam/runners/direct/EmptyInputProvider.java | 22 +-
.../direct/ExecutorServiceParallelExecutor.java | 12 +-
.../GroupAlsoByWindowEvaluatorFactory.java | 92 ++--
.../runners/direct/ParDoEvaluatorFactory.java | 56 +--
.../direct/ParDoMultiEvaluatorHooks.java | 55 ---
.../direct/ParDoMultiOverrideFactory.java | 51 +++
.../runners/direct/ParDoOverrideFactory.java | 53 ---
.../direct/ParDoSingleEvaluatorHooks.java | 58 ---
.../ParDoSingleViaMultiOverrideFactory.java | 66 +++
.../beam/runners/direct/RootInputProvider.java | 16 +-
.../runners/direct/RootProviderRegistry.java | 19 +-
.../runners/direct/StepTransformResult.java | 6 +
.../direct/TestStreamEvaluatorFactory.java | 23 +-
.../direct/TransformEvaluatorRegistry.java | 12 +-
.../beam/runners/direct/TransformExecutor.java | 28 +-
.../direct/UnboundedReadEvaluatorFactory.java | 32 +-
.../direct/BoundedReadEvaluatorFactoryTest.java | 186 +++++++-
.../direct/ParDoMultiEvaluatorHooksTest.java | 439 -------------------
.../direct/ParDoSingleEvaluatorHooksTest.java | 335 --------------
.../runners/direct/TransformExecutorTest.java | 43 --
runners/flink/runner/pom.xml | 1 +
.../FlinkBatchTransformTranslators.java | 34 +-
.../FlinkStreamingTransformTranslators.java | 38 +-
.../functions/FlinkDoFnFunction.java | 21 +-
.../FlinkMergingNonShuffleReduceFunction.java | 14 +-
.../FlinkMergingPartialReduceFunction.java | 14 +-
.../functions/FlinkMergingReduceFunction.java | 12 +-
.../functions/FlinkMultiOutputDoFnFunction.java | 14 +-
.../FlinkMultiOutputProcessContext.java | 94 +---
.../functions/FlinkPartialReduceFunction.java | 14 +-
.../functions/FlinkProcessContext.java | 330 --------------
.../functions/FlinkProcessContextBase.java | 278 ++++++++++++
.../functions/FlinkReduceFunction.java | 14 +-
.../FlinkSingleOutputProcessContext.java | 69 +++
runners/google-cloud-dataflow-java/pom.xml | 2 +-
.../beam/runners/dataflow/DataflowRunner.java | 2 +-
.../dataflow/util/DataflowTransport.java | 10 +-
.../options/DataflowPipelineOptionsTest.java | 6 +-
.../runners/dataflow/util/PackageUtilTest.java | 2 +-
runners/spark/pom.xml | 4 +-
.../runners/spark/SparkPipelineOptions.java | 10 +-
.../apache/beam/runners/spark/SparkRunner.java | 9 +-
.../spark/translation/BoundedDataset.java | 114 +++++
.../beam/runners/spark/translation/Dataset.java | 34 ++
.../runners/spark/translation/DoFnFunction.java | 10 +-
.../spark/translation/EvaluationContext.java | 257 +++++------
.../spark/translation/MultiDoFnFunction.java | 21 +-
.../spark/translation/SparkProcessContext.java | 81 ++--
.../spark/translation/SparkRuntimeContext.java | 4 +-
.../spark/translation/TransformTranslator.java | 122 +++---
.../SparkRunnerStreamingContextFactory.java | 8 +-
.../streaming/StreamingEvaluationContext.java | 272 ------------
.../streaming/StreamingTransformTranslator.java | 135 +++---
.../translation/streaming/UnboundedDataset.java | 103 +++++
.../runners/spark/ProvidedSparkContextTest.java | 24 +-
.../metrics/sink/NamedAggregatorsTest.java | 13 +-
.../beam/runners/spark/io/AvroPipelineTest.java | 11 +-
.../beam/runners/spark/io/NumShardsTest.java | 11 +-
.../io/hadoop/HadoopFileFormatPipelineTest.java | 11 +-
.../spark/translation/SideEffectsTest.java | 41 +-
.../streaming/EmptyStreamAssertionTest.java | 8 +-
.../streaming/FlattenStreamingTest.java | 9 +-
.../streaming/KafkaStreamingTest.java | 16 +-
.../ResumeFromCheckpointStreamingTest.java | 8 +-
.../streaming/SimpleStreamingWordCountTest.java | 11 +-
.../streaming/utils/PAssertStreaming.java | 8 +-
.../utils/SparkTestPipelineOptions.java | 42 ++
.../SparkTestPipelineOptionsForStreaming.java | 43 ++
.../utils/TestOptionsForStreaming.java | 55 ---
.../src/main/resources/beam/findbugs-filter.xml | 172 +++++++-
sdks/java/core/pom.xml | 2 +-
.../main/java/org/apache/beam/sdk/Pipeline.java | 17 -
.../org/apache/beam/sdk/coders/AvroCoder.java | 5 -
.../apache/beam/sdk/coders/InstantCoder.java | 4 -
.../org/apache/beam/sdk/io/FileBasedSink.java | 197 +--------
.../beam/sdk/metrics/MetricsContainer.java | 13 -
.../beam/sdk/metrics/MetricsEnvironment.java | 60 ++-
.../sdk/options/PipelineOptionsFactory.java | 32 +-
.../apache/beam/sdk/runners/PipelineRunner.java | 2 +-
.../org/apache/beam/sdk/testing/PAssert.java | 3 -
.../beam/sdk/testing/SerializableMatchers.java | 3 -
.../apache/beam/sdk/testing/TestPipeline.java | 2 +-
.../beam/sdk/testing/UsesStatefulParDo.java | 25 ++
.../org/apache/beam/sdk/transforms/DoFn.java | 109 +----
.../beam/sdk/transforms/DoFnAdapters.java | 42 +-
.../apache/beam/sdk/transforms/DoFnTester.java | 18 +-
.../org/apache/beam/sdk/transforms/OldDoFn.java | 7 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 2 +-
.../beam/sdk/transforms/WithTimestamps.java | 5 +-
.../reflect/ByteBuddyDoFnInvokerFactory.java | 325 +++++++-------
.../reflect/ByteBuddyOnTimerInvokerFactory.java | 42 +-
.../sdk/transforms/reflect/DoFnInvoker.java | 123 +++++-
.../sdk/transforms/reflect/DoFnInvokers.java | 14 +-
.../sdk/transforms/reflect/DoFnSignature.java | 52 ++-
.../sdk/transforms/reflect/DoFnSignatures.java | 61 ++-
.../sdk/transforms/reflect/OnTimerInvoker.java | 10 +-
...AttemptAndTimeBoundedExponentialBackOff.java | 3 -
.../sdk/util/ExposedByteArrayInputStream.java | 2 -
.../sdk/util/ExposedByteArrayOutputStream.java | 3 -
.../beam/sdk/util/FileIOChannelFactory.java | 67 ++-
.../sdk/util/FileIOChannelFactoryRegistrar.java | 38 ++
.../beam/sdk/util/GcpCredentialFactory.java | 13 +-
.../beam/sdk/util/GcsIOChannelFactory.java | 20 +-
.../sdk/util/GcsIOChannelFactoryRegistrar.java | 38 ++
.../apache/beam/sdk/util/IOChannelFactory.java | 22 +
.../sdk/util/IOChannelFactoryRegistrar.java | 48 ++
.../apache/beam/sdk/util/IOChannelUtils.java | 134 +++++-
.../beam/sdk/util/NoopCredentialFactory.java | 31 +-
.../sdk/util/NullCredentialInitializer.java | 62 +++
.../apache/beam/sdk/util/TimerInternals.java | 13 +-
.../org/apache/beam/sdk/util/Transport.java | 9 +-
.../beam/sdk/util/WindowingInternals.java | 14 +-
.../java/org/apache/beam/sdk/util/ZipFiles.java | 9 -
.../beam/sdk/util/common/ReflectHelpers.java | 29 ++
.../beam/sdk/util/state/StateContexts.java | 20 +-
.../beam/sdk/util/state/StateMerging.java | 3 -
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 2 +-
.../apache/beam/sdk/io/FileBasedSourceTest.java | 2 +-
.../java/org/apache/beam/sdk/io/TextIOTest.java | 2 +-
.../sdk/metrics/MetricsEnvironmentTest.java | 8 +-
.../apache/beam/sdk/metrics/MetricsTest.java | 6 +-
.../sdk/options/PipelineOptionsFactoryTest.java | 34 --
.../apache/beam/sdk/transforms/OldDoFnTest.java | 11 +-
.../apache/beam/sdk/transforms/ParDoTest.java | 55 ++-
.../transforms/reflect/DoFnInvokersTest.java | 228 ++++++++--
.../DoFnSignaturesProcessElementTest.java | 22 -
.../DoFnSignaturesSplittableDoFnTest.java | 2 +-
.../transforms/reflect/DoFnSignaturesTest.java | 25 +-
.../transforms/reflect/OnTimerInvokersTest.java | 7 +-
.../testhelper/DoFnInvokersTestHelper.java | 137 ++++++
.../util/FileIOChannelFactoryRegistrarTest.java | 44 ++
.../beam/sdk/util/FileIOChannelFactoryTest.java | 10 +-
.../util/GcsIOChannelFactoryRegistrarTest.java | 44 ++
.../beam/sdk/util/GcsIOChannelFactoryTest.java | 2 +-
.../beam/sdk/util/IOChannelUtilsTest.java | 39 ++
.../sdk/util/common/ReflectHelpersTest.java | 33 ++
sdks/java/io/google-cloud-platform/pom.xml | 14 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 4 +
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 6 +-
sdks/java/io/hdfs/pom.xml | 2 +-
sdks/java/io/jdbc/pom.xml | 2 +-
sdks/java/io/jms/pom.xml | 2 +-
sdks/java/io/kafka/pom.xml | 2 +-
sdks/java/io/kinesis/pom.xml | 2 +-
sdks/java/io/mongodb/pom.xml | 15 +-
.../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 308 +++++++++++--
.../apache/beam/sdk/io/mongodb/MongoDbIO.java | 8 +-
.../sdk/io/mongodb/MongoDBGridFSIOTest.java | 121 ++++-
.../beam/sdk/io/mongodb/MongoDbIOTest.java | 38 +-
sdks/java/maven-archetypes/examples/pom.xml | 42 ++
.../main/resources/archetype-resources/pom.xml | 14 +
sdks/java/maven-archetypes/starter/pom.xml | 6 +
.../main/resources/archetype-resources/pom.xml | 14 +
.../src/main/java/StarterPipeline.java | 18 +-
.../resources/projects/basic/reference/pom.xml | 14 +
.../src/main/java/it/pkg/StarterPipeline.java | 18 +-
sdks/java/microbenchmarks/README.md | 42 --
sdks/java/microbenchmarks/pom.xml | 113 -----
.../coders/AvroCoderBenchmark.java | 119 -----
.../coders/ByteArrayCoderBenchmark.java | 64 ---
.../coders/CoderBenchmarking.java | 41 --
.../coders/StringUtf8CoderBenchmark.java | 70 ---
.../microbenchmarks/coders/package-info.java | 22 -
.../transforms/DoFnInvokersBenchmark.java | 231 ----------
.../transforms/package-info.java | 22 -
sdks/java/pom.xml | 1 -
192 files changed, 4606 insertions(+), 4350 deletions(-)
----------------------------------------------------------------------