You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2016/11/22 20:11:54 UTC
[50/50] incubator-beam git commit: This closes #1416
This closes #1416
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/b4187bd9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/b4187bd9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/b4187bd9
Branch: refs/heads/python-sdk
Commit: b4187bd91e9e53c1562ee845ffa87cc9e734006f
Parents: 28bfd90 0a66721
Author: Kenneth Knowles <kl...@google.com>
Authored: Tue Nov 22 12:08:38 2016 -0800
Committer: Kenneth Knowles <kl...@google.com>
Committed: Tue Nov 22 12:08:38 2016 -0800
----------------------------------------------------------------------
.travis.yml | 12 +-
README.md | 4 +-
examples/java/pom.xml | 185 ++--
.../beam/examples/DebuggingWordCount.java | 69 +-
.../apache/beam/examples/MinimalWordCount.java | 52 +-
.../org/apache/beam/examples/WordCount.java | 64 +-
.../beam/examples/common/ExampleUtils.java | 3 +-
.../apache/beam/examples/complete/TfIdf.java | 8 +-
.../examples/complete/TopWikipediaSessions.java | 2 +-
.../examples/cookbook/BigQueryTornadoes.java | 2 +-
.../cookbook/CombinePerKeyExamples.java | 2 +-
.../beam/examples/cookbook/DeDupExample.java | 10 +-
.../beam/examples/cookbook/FilterExamples.java | 2 +-
.../beam/examples/cookbook/JoinExamples.java | 2 +-
.../examples/cookbook/MaxPerKeyExamples.java | 2 +-
.../org/apache/beam/examples/cookbook/README.md | 2 +-
.../beam/examples/DebuggingWordCountTest.java | 15 +-
.../beam/examples/WindowedWordCountIT.java | 31 +-
.../org/apache/beam/examples/WordCountIT.java | 21 +-
.../org/apache/beam/examples/WordCountTest.java | 2 +-
.../examples/complete/AutoCompleteTest.java | 6 +-
.../beam/examples/complete/TfIdfTest.java | 6 +-
.../complete/TopWikipediaSessionsTest.java | 2 +-
.../examples/cookbook/BigQueryTornadoesIT.java | 15 +-
.../examples/cookbook/DeDupExampleTest.java | 82 --
.../examples/cookbook/DistinctExampleTest.java | 82 ++
.../examples/cookbook/JoinExamplesTest.java | 2 +-
.../examples/cookbook/TriggerExampleTest.java | 2 +-
examples/java8/pom.xml | 32 +-
.../beam/examples/MinimalWordCountJava8.java | 2 +-
.../beam/examples/complete/game/GameStats.java | 11 +-
.../examples/complete/game/HourlyTeamScore.java | 10 +-
.../examples/complete/game/LeaderBoard.java | 12 +-
.../beam/examples/complete/game/UserScore.java | 10 +-
.../examples/MinimalWordCountJava8Test.java | 2 +-
.../examples/complete/game/GameStatsTest.java | 7 +-
.../complete/game/HourlyTeamScoreTest.java | 7 +-
.../examples/complete/game/LeaderBoardTest.java | 16 +-
.../examples/complete/game/UserScoreTest.java | 6 +-
examples/pom.xml | 18 +-
pom.xml | 168 +++-
runners/apex/README.md | 76 ++
runners/apex/pom.xml | 234 ++++++
.../beam/runners/apex/ApexPipelineOptions.java | 60 ++
.../apache/beam/runners/apex/ApexRunner.java | 398 +++++++++
.../beam/runners/apex/ApexRunnerRegistrar.java | 61 ++
.../beam/runners/apex/ApexRunnerResult.java | 110 +++
.../beam/runners/apex/TestApexRunner.java | 73 ++
.../apache/beam/runners/apex/package-info.java | 22 +
.../translation/ApexPipelineTranslator.java | 179 ++++
.../translation/CreateValuesTranslator.java | 48 ++
.../FlattenPCollectionTranslator.java | 129 +++
.../apex/translation/GroupByKeyTranslator.java | 42 +
.../translation/ParDoBoundMultiTranslator.java | 142 ++++
.../apex/translation/ParDoBoundTranslator.java | 64 ++
.../translation/ReadUnboundedTranslator.java | 42 +
.../apex/translation/TransformTranslator.java | 31 +
.../apex/translation/TranslationContext.java | 178 ++++
.../operators/ApexFlattenOperator.java | 125 +++
.../operators/ApexGroupByKeyOperator.java | 489 +++++++++++
.../operators/ApexParDoOperator.java | 375 +++++++++
.../ApexReadUnboundedInputOperator.java | 155 ++++
.../translation/operators/package-info.java | 22 +
.../runners/apex/translation/package-info.java | 22 +
.../translation/utils/ApexStateInternals.java | 438 ++++++++++
.../apex/translation/utils/ApexStreamTuple.java | 222 +++++
.../utils/CoderAdapterStreamCodec.java | 69 ++
.../apex/translation/utils/NoOpStepContext.java | 72 ++
.../utils/SerializablePipelineOptions.java | 60 ++
.../utils/ValueAndCoderKryoSerializable.java | 77 ++
.../apex/translation/utils/ValuesSource.java | 149 ++++
.../apex/translation/utils/package-info.java | 22 +
.../runners/apex/ApexRunnerRegistrarTest.java | 47 ++
.../apex/examples/UnboundedTextSource.java | 142 ++++
.../runners/apex/examples/WordCountTest.java | 188 +++++
.../runners/apex/examples/package-info.java | 22 +
.../translation/ApexGroupByKeyOperatorTest.java | 117 +++
.../FlattenPCollectionTranslatorTest.java | 99 +++
.../translation/GroupByKeyTranslatorTest.java | 246 ++++++
.../translation/ParDoBoundTranslatorTest.java | 340 ++++++++
.../translation/ReadUnboundTranslatorTest.java | 129 +++
.../utils/ApexStateInternalsTest.java | 361 ++++++++
.../translation/utils/CollectionSource.java | 136 +++
.../translation/utils/PipelineOptionsTest.java | 84 ++
.../apex/src/test/resources/log4j.properties | 35 +
runners/apex/src/test/resources/words.txt | 3 +
runners/core-java/pom.xml | 67 +-
.../core/GroupAlsoByWindowViaWindowSetDoFn.java | 6 +-
.../GroupAlsoByWindowsViaOutputBufferDoFn.java | 8 +-
.../core/GroupByKeyViaGroupByKeyOnly.java | 13 +-
.../beam/runners/core/OutputWindowedValue.java | 45 +
.../beam/runners/core/PaneInfoTracker.java | 3 -
.../core/PushbackSideInputDoFnRunner.java | 28 +-
.../runners/core/ReduceFnContextFactory.java | 83 +-
.../beam/runners/core/ReduceFnRunner.java | 40 +-
.../beam/runners/core/SimpleDoFnRunner.java | 158 ++--
.../beam/runners/core/SimpleOldDoFnRunner.java | 21 +-
.../beam/runners/core/SplittableParDo.java | 60 +-
.../beam/runners/core/SystemReduceFn.java | 3 -
.../core/UnboundedReadFromBoundedSource.java | 3 +-
.../apache/beam/runners/core/WatermarkHold.java | 3 -
.../core/WindowingInternalsAdapters.java | 76 ++
.../AfterDelayFromFirstElementStateMachine.java | 5 -
.../core/triggers/AfterPaneStateMachine.java | 3 -
.../core/triggers/TriggerStateMachines.java | 1 -
.../beam/runners/core/ReduceFnTester.java | 87 +-
.../UnboundedReadFromBoundedSourceTest.java | 4 +-
runners/direct-java/pom.xml | 104 +--
.../runners/direct/AggregatorContainer.java | 22 +-
.../direct/BoundedReadEvaluatorFactory.java | 110 ++-
...ectGBKIntoKeyedWorkItemsOverrideFactory.java | 16 +-
.../beam/runners/direct/DirectGroupByKey.java | 42 +-
.../direct/DirectGroupByKeyOverrideFactory.java | 25 +-
.../beam/runners/direct/DirectRunner.java | 159 ++--
.../runners/direct/DirectTimerInternals.java | 13 +
.../runners/direct/DoFnLifecycleManager.java | 4 +-
.../beam/runners/direct/EmptyInputProvider.java | 22 +-
.../direct/EncodabilityEnforcementFactory.java | 80 --
.../beam/runners/direct/EvaluationContext.java | 18 +-
.../direct/ExecutorServiceParallelExecutor.java | 107 +--
.../GroupAlsoByWindowEvaluatorFactory.java | 227 +++--
.../direct/GroupByKeyOnlyEvaluatorFactory.java | 31 +-
.../ImmutabilityCheckingBundleFactory.java | 11 +-
.../direct/ImmutabilityEnforcementFactory.java | 2 -
.../direct/ImmutableListBundleFactory.java | 21 +-
.../direct/PTransformOverrideFactory.java | 8 +-
.../beam/runners/direct/ParDoEvaluator.java | 28 +-
.../runners/direct/ParDoEvaluatorFactory.java | 108 +++
.../direct/ParDoMultiEvaluatorFactory.java | 107 ---
.../direct/ParDoMultiOverrideFactory.java | 51 ++
.../runners/direct/ParDoOverrideFactory.java | 55 --
.../direct/ParDoSingleEvaluatorFactory.java | 110 ---
.../ParDoSingleViaMultiOverrideFactory.java | 66 ++
.../beam/runners/direct/RootInputProvider.java | 16 +-
.../runners/direct/RootProviderRegistry.java | 19 +-
.../runners/direct/StepTransformResult.java | 6 +
.../direct/TestStreamEvaluatorFactory.java | 37 +-
.../direct/TransformEvaluatorRegistry.java | 10 +-
.../beam/runners/direct/TransformExecutor.java | 23 -
.../direct/UnboundedReadEvaluatorFactory.java | 38 +-
.../direct/UncommittedBundleOutputManager.java | 50 --
.../runners/direct/ViewEvaluatorFactory.java | 19 +-
.../direct/WatermarkCallbackExecutor.java | 7 +-
.../beam/runners/direct/WatermarkManager.java | 244 ++++--
.../direct/WriteWithShardingFactory.java | 15 +-
.../direct/BoundedReadEvaluatorFactoryTest.java | 186 ++++-
.../direct/CloningBundleFactoryTest.java | 122 ++-
.../ConsumerTrackingPipelineVisitorTest.java | 17 +
.../beam/runners/direct/DirectRunnerTest.java | 88 ++
.../EncodabilityEnforcementFactoryTest.java | 323 -------
.../runners/direct/EvaluationContextTest.java | 23 +-
.../direct/GroupByKeyEvaluatorFactoryTest.java | 29 +-
.../GroupByKeyOnlyEvaluatorFactoryTest.java | 31 +-
.../direct/ImmutableListBundleFactoryTest.java | 15 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 16 +-
.../direct/ParDoMultiEvaluatorFactoryTest.java | 439 ----------
.../direct/ParDoSingleEvaluatorFactoryTest.java | 335 --------
.../runners/direct/TransformExecutorTest.java | 43 -
.../UnboundedReadEvaluatorFactoryTest.java | 47 +-
.../runners/direct/WatermarkManagerTest.java | 185 ++--
.../direct/WriteWithShardingFactoryTest.java | 26 +-
runners/flink/examples/pom.xml | 17 -
.../beam/runners/flink/examples/TFIDF.java | 17 +-
.../flink/examples/streaming/AutoComplete.java | 8 +-
.../flink/examples/streaming/JoinExamples.java | 12 +-
.../examples/streaming/KafkaIOExamples.java | 2 +-
.../examples/streaming/WindowedWordCount.java | 8 +-
runners/flink/pom.xml | 1 +
runners/flink/runner/pom.xml | 17 -
.../apache/beam/runners/flink/FlinkRunner.java | 4 +-
.../FlinkStreamingTransformTranslators.java | 17 +-
.../functions/FlinkDoFnFunction.java | 21 +-
.../FlinkMergingNonShuffleReduceFunction.java | 14 +-
.../FlinkMergingPartialReduceFunction.java | 14 +-
.../functions/FlinkMergingReduceFunction.java | 12 +-
.../functions/FlinkMultiOutputDoFnFunction.java | 14 +-
.../FlinkMultiOutputProcessContext.java | 94 +--
.../functions/FlinkPartialReduceFunction.java | 14 +-
.../functions/FlinkProcessContext.java | 330 --------
.../functions/FlinkProcessContextBase.java | 278 ++++++
.../functions/FlinkReduceFunction.java | 14 +-
.../FlinkSingleOutputProcessContext.java | 69 ++
.../SerializableFnAggregatorWrapper.java | 7 +
.../wrappers/streaming/DoFnOperator.java | 10 +-
.../wrappers/streaming/WindowDoFnOperator.java | 15 +-
.../streaming/io/UnboundedSocketSource.java | 5 +-
.../streaming/io/UnboundedSourceWrapper.java | 3 +-
runners/google-cloud-dataflow-java/pom.xml | 144 +---
.../dataflow/BlockingDataflowRunner.java | 170 ----
.../dataflow/DataflowJobCancelledException.java | 39 -
.../dataflow/DataflowJobExecutionException.java | 35 -
.../dataflow/DataflowJobUpdatedException.java | 51 --
.../runners/dataflow/DataflowPipelineJob.java | 44 +-
.../dataflow/DataflowPipelineRegistrar.java | 11 +-
.../dataflow/DataflowPipelineTranslator.java | 8 +-
.../beam/runners/dataflow/DataflowRunner.java | 51 +-
.../runners/dataflow/internal/IsmFormat.java | 5 +-
.../dataflow/internal/ReadTranslator.java | 12 +-
.../BlockingDataflowPipelineOptions.java | 28 -
.../options/DataflowPipelineOptions.java | 8 +
.../testing/TestDataflowPipelineOptions.java | 6 +-
.../dataflow/testing/TestDataflowRunner.java | 27 +-
.../dataflow/util/DataflowTemplateJob.java | 70 ++
.../dataflow/util/DataflowTransport.java | 13 +-
.../beam/runners/dataflow/util/DoFnInfo.java | 53 +-
.../runners/dataflow/util/MonitoringUtil.java | 1 -
.../runners/dataflow/util/RandomAccessData.java | 2 +-
.../dataflow/BlockingDataflowRunnerTest.java | 300 -------
.../dataflow/DataflowPipelineJobTest.java | 30 +-
.../dataflow/DataflowPipelineRegistrarTest.java | 7 +-
.../DataflowPipelineTranslatorTest.java | 28 +
.../runners/dataflow/DataflowRunnerTest.java | 45 +
.../options/DataflowPipelineOptionsTest.java | 6 +-
.../runners/dataflow/util/PackageUtilTest.java | 2 +-
runners/pom.xml | 17 +
runners/spark/pom.xml | 24 +-
.../beam/runners/spark/SparkContextOptions.java | 64 ++
.../runners/spark/SparkPipelineOptions.java | 49 +-
.../apache/beam/runners/spark/SparkRunner.java | 9 +-
.../apache/beam/runners/spark/io/SourceRDD.java | 4 +-
.../spark/translation/BoundedDataset.java | 114 +++
.../beam/runners/spark/translation/Dataset.java | 34 +
.../runners/spark/translation/DoFnFunction.java | 10 +-
.../spark/translation/EvaluationContext.java | 257 +++---
.../translation/GroupCombineFunctions.java | 5 +-
.../spark/translation/MultiDoFnFunction.java | 21 +-
.../ReifyTimestampsAndWindowsFunction.java | 47 ++
.../spark/translation/SparkContextFactory.java | 19 +-
.../spark/translation/SparkProcessContext.java | 125 +--
.../spark/translation/SparkRuntimeContext.java | 4 +-
.../spark/translation/TransformTranslator.java | 99 +--
.../SparkRunnerStreamingContextFactory.java | 34 +-
.../streaming/StreamingEvaluationContext.java | 272 ------
.../streaming/StreamingTransformTranslator.java | 135 ++-
.../translation/streaming/UnboundedDataset.java | 103 +++
.../runners/spark/ProvidedSparkContextTest.java | 24 +-
.../metrics/sink/NamedAggregatorsTest.java | 13 +-
.../beam/runners/spark/io/AvroPipelineTest.java | 11 +-
.../beam/runners/spark/io/NumShardsTest.java | 11 +-
.../io/hadoop/HadoopFileFormatPipelineTest.java | 11 +-
.../spark/translation/SideEffectsTest.java | 41 +-
.../streaming/EmptyStreamAssertionTest.java | 11 +-
.../streaming/FlattenStreamingTest.java | 15 +-
.../streaming/KafkaStreamingTest.java | 28 +-
.../ResumeFromCheckpointStreamingTest.java | 11 +-
.../streaming/SimpleStreamingWordCountTest.java | 12 +-
.../streaming/utils/PAssertStreaming.java | 8 +-
.../utils/SparkTestPipelineOptions.java | 42 +
.../SparkTestPipelineOptionsForStreaming.java | 43 +
.../utils/TestOptionsForStreaming.java | 55 --
sdks/java/build-tools/pom.xml | 4 +-
.../src/main/resources/beam/checkstyle.xml | 9 +-
.../src/main/resources/beam/findbugs-filter.xml | 172 +++-
sdks/java/core/pom.xml | 166 +---
.../main/java/org/apache/beam/sdk/Pipeline.java | 17 -
.../org/apache/beam/sdk/coders/AvroCoder.java | 121 ++-
.../apache/beam/sdk/coders/InstantCoder.java | 4 -
.../java/org/apache/beam/sdk/io/AvroIO.java | 4 +-
.../sdk/io/BoundedReadFromUnboundedSource.java | 4 +-
.../org/apache/beam/sdk/io/BoundedSource.java | 11 +-
.../apache/beam/sdk/io/CompressedSource.java | 4 +-
.../org/apache/beam/sdk/io/FileBasedSink.java | 295 ++-----
.../org/apache/beam/sdk/io/FileBasedSource.java | 59 +-
.../beam/sdk/io/PubsubUnboundedSource.java | 123 ++-
.../main/java/org/apache/beam/sdk/io/Sink.java | 4 +-
.../java/org/apache/beam/sdk/io/TextIO.java | 43 +-
.../java/org/apache/beam/sdk/io/XmlSource.java | 2 +-
.../apache/beam/sdk/io/range/ByteKeyRange.java | 2 +-
.../beam/sdk/metrics/MetricsContainer.java | 13 -
.../org/apache/beam/sdk/options/GcpOptions.java | 136 +--
.../sdk/options/PipelineOptionsFactory.java | 280 +++++--
.../sdk/options/ProxyInvocationHandler.java | 5 +-
.../apache/beam/sdk/options/ValueProvider.java | 62 +-
.../apache/beam/sdk/runners/PipelineRunner.java | 2 +-
.../beam/sdk/testing/BigqueryMatcher.java | 15 +-
.../org/apache/beam/sdk/testing/PAssert.java | 3 -
.../beam/sdk/testing/SerializableMatchers.java | 3 -
.../apache/beam/sdk/testing/TestPipeline.java | 2 +-
.../beam/sdk/testing/TestPipelineOptions.java | 6 +
.../apache/beam/sdk/transforms/Aggregator.java | 13 +-
.../apache/beam/sdk/transforms/CombineFns.java | 4 +-
.../sdk/transforms/DelegatingAggregator.java | 7 +-
.../apache/beam/sdk/transforms/Distinct.java | 158 ++++
.../org/apache/beam/sdk/transforms/DoFn.java | 111 +--
.../beam/sdk/transforms/DoFnAdapters.java | 70 +-
.../apache/beam/sdk/transforms/DoFnTester.java | 18 +-
.../org/apache/beam/sdk/transforms/Flatten.java | 3 +-
.../apache/beam/sdk/transforms/GroupByKey.java | 4 +-
.../org/apache/beam/sdk/transforms/OldDoFn.java | 7 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 51 +-
.../org/apache/beam/sdk/transforms/Regex.java | 454 ++++++++++
.../beam/sdk/transforms/RemoveDuplicates.java | 159 ----
.../org/apache/beam/sdk/transforms/View.java | 2 +-
.../beam/sdk/transforms/WithTimestamps.java | 5 +-
.../sdk/transforms/display/DisplayData.java | 5 +-
.../beam/sdk/transforms/join/CoGroupByKey.java | 6 +-
.../reflect/ByteBuddyDoFnInvokerFactory.java | 837 +++++++++++++++++++
.../reflect/ByteBuddyOnTimerInvokerFactory.java | 307 +++++++
.../sdk/transforms/reflect/DoFnInvoker.java | 123 ++-
.../transforms/reflect/DoFnInvokerFactory.java | 27 +
.../sdk/transforms/reflect/DoFnInvokers.java | 673 +--------------
.../sdk/transforms/reflect/DoFnSignature.java | 163 +++-
.../sdk/transforms/reflect/DoFnSignatures.java | 826 +++++++++++-------
.../sdk/transforms/reflect/OnTimerInvoker.java | 27 +
.../reflect/OnTimerInvokerFactory.java | 36 +
.../sdk/transforms/reflect/OnTimerInvokers.java | 42 +
.../splittabledofn/RestrictionTracker.java | 2 +-
.../transforms/splittabledofn/package-info.java | 4 +-
.../beam/sdk/transforms/windowing/Window.java | 130 +--
.../beam/sdk/transforms/windowing/WindowFn.java | 12 +
...AttemptAndTimeBoundedExponentialBackOff.java | 3 -
.../beam/sdk/util/BaseExecutionContext.java | 2 +-
.../BufferedElementCountingOutputStream.java | 4 +-
.../apache/beam/sdk/util/CredentialFactory.java | 4 +-
.../org/apache/beam/sdk/util/Credentials.java | 192 -----
.../apache/beam/sdk/util/ExecutableTrigger.java | 131 ---
.../apache/beam/sdk/util/ExecutionContext.java | 8 +-
.../sdk/util/ExposedByteArrayInputStream.java | 7 +-
.../sdk/util/ExposedByteArrayOutputStream.java | 5 -
.../beam/sdk/util/FileIOChannelFactory.java | 74 +-
.../sdk/util/FileIOChannelFactoryRegistrar.java | 38 +
.../apache/beam/sdk/util/GatherAllPanes.java | 18 +-
.../beam/sdk/util/GcpCredentialFactory.java | 42 +-
.../beam/sdk/util/GcsIOChannelFactory.java | 28 +-
.../sdk/util/GcsIOChannelFactoryRegistrar.java | 38 +
.../java/org/apache/beam/sdk/util/GcsUtil.java | 2 -
.../apache/beam/sdk/util/IOChannelFactory.java | 26 +
.../sdk/util/IOChannelFactoryRegistrar.java | 48 ++
.../apache/beam/sdk/util/IOChannelUtils.java | 134 ++-
.../apache/beam/sdk/util/MovingFunction.java | 4 +-
.../beam/sdk/util/NoopCredentialFactory.java | 40 +-
.../sdk/util/NullCredentialInitializer.java | 62 ++
.../org/apache/beam/sdk/util/PubsubClient.java | 32 +-
.../apache/beam/sdk/util/PubsubGrpcClient.java | 18 +-
.../apache/beam/sdk/util/PubsubJsonClient.java | 9 +-
.../apache/beam/sdk/util/PubsubTestClient.java | 32 +
.../sdk/util/ReifyTimestampAndWindowsDoFn.java | 41 -
.../sdk/util/ReifyTimestampsAndWindows.java | 63 --
.../org/apache/beam/sdk/util/StringUtils.java | 8 +-
.../apache/beam/sdk/util/TestCredential.java | 44 +-
.../apache/beam/sdk/util/TimerInternals.java | 48 +-
.../org/apache/beam/sdk/util/Transport.java | 18 +-
.../beam/sdk/util/WindowingInternals.java | 14 +-
.../apache/beam/sdk/util/WindowingStrategy.java | 10 +-
.../java/org/apache/beam/sdk/util/ZipFiles.java | 9 -
.../beam/sdk/util/common/ReflectHelpers.java | 46 +
.../org/apache/beam/sdk/util/gcsfs/GcsPath.java | 13 +-
.../sdk/util/state/InMemoryTimerInternals.java | 12 +
.../beam/sdk/util/state/StateContexts.java | 20 +-
.../beam/sdk/util/state/StateMerging.java | 3 -
.../beam/sdk/util/state/StateNamespace.java | 16 +-
.../java/org/apache/beam/sdk/values/PDone.java | 2 +-
.../apache/beam/sdk/values/TypeDescriptor.java | 48 +-
.../apache/beam/sdk/values/TypeDescriptors.java | 1 -
.../apache/beam/sdk/values/TypeParameter.java | 62 ++
.../apache/beam/sdk/coders/AvroCoderTest.java | 48 ++
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 2 +-
.../apache/beam/sdk/io/CountingInputTest.java | 6 +-
.../apache/beam/sdk/io/CountingSourceTest.java | 8 +-
.../apache/beam/sdk/io/FileBasedSinkTest.java | 102 +--
.../apache/beam/sdk/io/FileBasedSourceTest.java | 2 +-
.../beam/sdk/io/PubsubUnboundedSourceTest.java | 79 ++
.../java/org/apache/beam/sdk/io/TextIOTest.java | 2 +-
.../org/apache/beam/sdk/io/XmlSinkTest.java | 16 +-
.../sdk/options/PipelineOptionsFactoryTest.java | 273 +++++-
.../sdk/options/ProxyInvocationHandlerTest.java | 29 +-
.../beam/sdk/options/ValueProviderTest.java | 40 +
.../apache/beam/sdk/testing/ExpectedLogs.java | 15 +-
.../beam/sdk/testing/ExpectedLogsTest.java | 30 +-
.../apache/beam/sdk/testing/PAssertTest.java | 27 +
.../sdk/transforms/ApproximateUniqueTest.java | 2 +-
.../beam/sdk/transforms/DistinctTest.java | 130 +++
.../apache/beam/sdk/transforms/FlattenTest.java | 38 +-
.../apache/beam/sdk/transforms/OldDoFnTest.java | 11 +-
.../apache/beam/sdk/transforms/ParDoTest.java | 116 ++-
.../apache/beam/sdk/transforms/RegexTest.java | 248 ++++++
.../sdk/transforms/RemoveDuplicatesTest.java | 130 ---
.../transforms/reflect/DoFnInvokersTest.java | 328 ++++++--
.../DoFnSignaturesProcessElementTest.java | 52 +-
.../DoFnSignaturesSplittableDoFnTest.java | 60 +-
.../transforms/reflect/DoFnSignaturesTest.java | 132 +--
.../reflect/DoFnSignaturesTestUtils.java | 13 +-
.../transforms/reflect/OnTimerInvokersTest.java | 108 +++
.../testhelper/DoFnInvokersTestHelper.java | 137 +++
.../sdk/transforms/windowing/WindowTest.java | 6 +-
.../beam/sdk/util/ExecutableTriggerTest.java | 109 ---
.../util/FileIOChannelFactoryRegistrarTest.java | 44 +
.../beam/sdk/util/FileIOChannelFactoryTest.java | 10 +-
.../util/GcsIOChannelFactoryRegistrarTest.java | 44 +
.../beam/sdk/util/GcsIOChannelFactoryTest.java | 2 +-
.../beam/sdk/util/IOChannelUtilsTest.java | 39 +
.../beam/sdk/util/PubsubGrpcClientTest.java | 9 +-
.../sdk/util/common/ReflectHelpersTest.java | 58 ++
.../apache/beam/sdk/util/gcsfs/GcsPathTest.java | 25 +
.../beam/sdk/values/TypeDescriptorTest.java | 12 +
sdks/java/extensions/join-library/pom.xml | 8 -
sdks/java/extensions/pom.xml | 1 +
sdks/java/extensions/sorter/README.md | 42 +
sdks/java/extensions/sorter/pom.xml | 159 ++++
.../sorter/BufferedExternalSorter.java | 125 +++
.../sdk/extensions/sorter/ExternalSorter.java | 225 +++++
.../sdk/extensions/sorter/InMemorySorter.java | 166 ++++
.../beam/sdk/extensions/sorter/SortValues.java | 213 +++++
.../beam/sdk/extensions/sorter/Sorter.java | 47 ++
.../sdk/extensions/sorter/package-info.java | 23 +
.../sorter/BufferedExternalSorterTest.java | 177 ++++
.../extensions/sorter/ExternalSorterTest.java | 87 ++
.../extensions/sorter/InMemorySorterTest.java | 144 ++++
.../sdk/extensions/sorter/SortValuesTest.java | 128 +++
.../sdk/extensions/sorter/SorterTestUtils.java | 129 +++
sdks/java/io/google-cloud-platform/pom.xml | 31 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 10 +-
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 98 ++-
.../sdk/io/gcp/bigtable/BigtableService.java | 6 +
.../io/gcp/bigtable/BigtableServiceImpl.java | 5 +
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 27 +-
.../apache/beam/sdk/io/gcp/ApiSurfaceTest.java | 132 +++
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 6 +-
.../sdk/io/gcp/bigtable/BigtableIOTest.java | 83 +-
.../sdk/io/gcp/bigtable/BigtableWriteIT.java | 26 +-
.../beam/sdk/io/gcp/datastore/V1ReadIT.java | 15 +-
.../sdk/io/gcp/datastore/V1TestOptions.java | 5 -
.../beam/sdk/io/gcp/datastore/V1TestUtil.java | 35 +-
.../beam/sdk/io/gcp/datastore/V1WriteIT.java | 9 +-
sdks/java/io/hdfs/pom.xml | 10 +-
sdks/java/io/jdbc/pom.xml | 16 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 39 +
.../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 58 +-
sdks/java/io/jms/pom.xml | 23 +-
sdks/java/io/kafka/pom.xml | 30 +-
.../beam/sdk/io/kafka/KafkaCheckpointMark.java | 32 +-
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 18 +-
.../apache/beam/sdk/io/kafka/KafkaIOTest.java | 6 +-
sdks/java/io/kinesis/pom.xml | 25 +-
.../beam/sdk/io/kinesis/CustomOptional.java | 21 +-
.../apache/beam/sdk/io/kinesis/KinesisIO.java | 35 +-
.../beam/sdk/io/kinesis/KinesisRecord.java | 3 +-
.../beam/sdk/io/kinesis/CustomOptionalTest.java | 10 +
sdks/java/io/mongodb/pom.xml | 14 +-
.../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 308 ++++++-
.../apache/beam/sdk/io/mongodb/MongoDbIO.java | 8 +-
.../sdk/io/mongodb/MongoDBGridFSIOTest.java | 121 ++-
.../beam/sdk/io/mongodb/MongoDbIOTest.java | 38 +-
sdks/java/java8tests/pom.xml | 11 -
.../beam/sdk/transforms/DistinctJava8Test.java | 104 +++
.../transforms/RemoveDuplicatesJava8Test.java | 104 ---
sdks/java/maven-archetypes/examples/pom.xml | 42 +
.../main/resources/archetype-resources/pom.xml | 46 +-
.../src/main/java/DebuggingWordCount.java | 69 +-
.../src/main/java/MinimalWordCount.java | 52 +-
.../src/main/java/WindowedWordCount.java | 6 +-
.../src/main/java/WordCount.java | 64 +-
.../common/ExampleBigQueryTableOptions.java | 2 +-
.../src/main/java/common/ExampleOptions.java | 5 +
...xamplePubsubTopicAndSubscriptionOptions.java | 2 +-
.../java/common/ExamplePubsubTopicOptions.java | 2 +-
.../src/main/java/common/ExampleUtils.java | 3 +-
.../src/test/java/DebuggingWordCountTest.java | 15 +-
.../src/test/java/WordCountTest.java | 7 +-
sdks/java/maven-archetypes/starter/pom.xml | 6 +
.../main/resources/archetype-resources/pom.xml | 14 +
.../resources/projects/basic/reference/pom.xml | 14 +
.../update-examples-archetype.sh | 59 ++
sdks/java/microbenchmarks/README.md | 42 -
sdks/java/microbenchmarks/pom.xml | 110 ---
.../coders/AvroCoderBenchmark.java | 119 ---
.../coders/ByteArrayCoderBenchmark.java | 64 --
.../coders/CoderBenchmarking.java | 41 -
.../coders/StringUtf8CoderBenchmark.java | 70 --
.../microbenchmarks/coders/package-info.java | 22 -
.../transforms/DoFnInvokersBenchmark.java | 231 -----
.../transforms/package-info.java | 22 -
sdks/java/pom.xml | 10 +-
sdks/pom.xml | 19 +
474 files changed, 20040 insertions(+), 9983 deletions(-)
----------------------------------------------------------------------