You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2017/01/29 16:22:36 UTC
[50/50] beam git commit: Closes #1861
Closes #1861
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/27cf68ee
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/27cf68ee
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/27cf68ee
Branch: refs/heads/python-sdk
Commit: 27cf68ee72bd58475c170712f7afe20102601606
Parents: 1bc6859 f1b8679
Author: Dan Halperin <dh...@google.com>
Authored: Sun Jan 29 08:21:18 2017 -0800
Committer: Dan Halperin <dh...@google.com>
Committed: Sun Jan 29 08:21:18 2017 -0800
----------------------------------------------------------------------
.jenkins/common_job_properties.groovy | 9 +-
...job_beam_PostCommit_Java_MavenInstall.groovy | 2 +-
.../job_beam_PreCommit_Java_MavenInstall.groovy | 2 +-
.../job_beam_Release_NightlySnapshot.groovy | 2 +-
.jenkins/job_seed.groovy | 2 +-
.travis/README.md | 2 +-
DISCLAIMER | 10 -
NOTICE | 4 +-
README.md | 46 +-
examples/java/README.md | 16 +-
examples/java/pom.xml | 21 +-
.../beam/examples/DebuggingWordCount.java | 4 +-
.../org/apache/beam/examples/WordCount.java | 6 +-
.../beam/examples/complete/AutoComplete.java | 2 +-
.../org/apache/beam/examples/complete/README.md | 14 +-
.../apache/beam/examples/complete/TfIdf.java | 2 +-
.../examples/complete/TopWikipediaSessions.java | 2 +-
.../examples/complete/TrafficMaxLaneFlow.java | 2 +-
.../beam/examples/complete/TrafficRoutes.java | 2 +-
.../examples/cookbook/BigQueryTornadoes.java | 2 +-
.../cookbook/CombinePerKeyExamples.java | 2 +-
.../org/apache/beam/examples/cookbook/README.md | 14 +-
.../beam/examples/cookbook/TriggerExample.java | 4 +-
.../beam/examples/WindowedWordCountIT.java | 16 +-
examples/java8/pom.xml | 2 +-
.../beam/examples/complete/game/GameStats.java | 7 +-
.../examples/complete/game/LeaderBoard.java | 5 +-
.../beam/examples/complete/game/UserScore.java | 2 +-
examples/pom.xml | 16 +-
pom.xml | 41 +-
runners/apex/README.md | 4 +-
runners/apex/pom.xml | 3 +-
.../beam/runners/apex/ApexPipelineOptions.java | 7 +-
.../apache/beam/runners/apex/ApexRunner.java | 43 +-
.../beam/runners/apex/ApexYarnLauncher.java | 23 +-
.../translation/CreateValuesTranslator.java | 18 +-
.../FlattenPCollectionTranslator.java | 28 +-
.../apex/translation/GroupByKeyTranslator.java | 2 +-
.../translation/ParDoBoundMultiTranslator.java | 27 +-
.../apex/translation/ParDoBoundTranslator.java | 4 +-
.../apex/translation/TranslationContext.java | 27 +-
.../apex/translation/WindowBoundTranslator.java | 8 +-
.../operators/ApexGroupByKeyOperator.java | 4 +-
.../operators/ApexParDoOperator.java | 6 +-
.../ApexReadUnboundedInputOperator.java | 17 +-
.../beam/runners/apex/ApexRunnerTest.java | 75 ++
.../beam/runners/apex/ApexYarnLauncherTest.java | 9 +-
.../runners/apex/examples/WordCountTest.java | 2 +-
.../translation/ParDoBoundTranslatorTest.java | 6 +-
.../translation/ReadUnboundTranslatorTest.java | 8 +-
.../utils/ApexStateInternalsTest.java | 2 +-
.../test/resources/beam-runners-apex.properties | 20 +
runners/core-java/pom.xml | 2 +-
.../beam/runners/core/AssignWindowsDoFn.java | 3 +-
.../apache/beam/runners/core/DoFnAdapters.java | 343 ++++++
.../apache/beam/runners/core/DoFnRunner.java | 21 -
.../apache/beam/runners/core/DoFnRunners.java | 138 +--
.../core/GroupAlsoByWindowViaWindowSetDoFn.java | 10 +-
.../runners/core/GroupAlsoByWindowsDoFn.java | 5 +-
.../beam/runners/core/KeyedWorkItemCoder.java | 4 +-
.../core/LateDataDroppingDoFnRunner.java | 1 -
.../apache/beam/runners/core/NonEmptyPanes.java | 2 +-
.../org/apache/beam/runners/core/OldDoFn.java | 472 ++++++++
.../runners/core/PerKeyCombineFnRunner.java | 70 --
.../runners/core/PerKeyCombineFnRunners.java | 101 --
.../beam/runners/core/SimpleDoFnRunner.java | 63 -
.../beam/runners/core/SimpleOldDoFnRunner.java | 7 +-
.../beam/runners/core/SplittableParDo.java | 7 -
.../core/UnboundedReadFromBoundedSource.java | 14 +-
.../AfterDelayFromFirstElementStateMachine.java | 2 +-
.../core/triggers/AfterPaneStateMachine.java | 2 +-
.../core/DoFnDelegatingAggregatorTest.java | 144 +++
.../core/GroupAlsoByWindowsProperties.java | 2 +-
.../runners/core/KeyedWorkItemCoderTest.java | 6 +
.../core/LateDataDroppingDoFnRunnerTest.java | 2 +-
.../apache/beam/runners/core/NoOpOldDoFn.java | 72 ++
.../beam/runners/core/OldDoFnContextTest.java | 72 ++
.../apache/beam/runners/core/OldDoFnTest.java | 192 +++
.../beam/runners/core/ReduceFnRunnerTest.java | 12 +-
.../beam/runners/core/ReduceFnTester.java | 2 +-
.../runners/core/SimpleOldDoFnRunnerTest.java | 2 +-
.../UnboundedReadFromBoundedSourceTest.java | 12 +-
runners/direct-java/pom.xml | 3 +-
.../direct/BoundedReadEvaluatorFactory.java | 10 +-
...ecycleManagerRemovingTransformEvaluator.java | 19 +-
.../beam/runners/direct/EvaluationContext.java | 2 +-
.../direct/ExecutorServiceParallelExecutor.java | 4 +-
.../runners/direct/FlattenEvaluatorFactory.java | 4 +-
.../GroupAlsoByWindowEvaluatorFactory.java | 9 +-
.../direct/GroupByKeyOnlyEvaluatorFactory.java | 10 +-
.../beam/runners/direct/ParDoEvaluator.java | 16 +-
.../runners/direct/ParDoEvaluatorFactory.java | 19 +-
.../direct/ParDoMultiOverrideFactory.java | 13 +-
.../direct/StatefulParDoEvaluatorFactory.java | 27 +-
.../direct/TestStreamEvaluatorFactory.java | 5 +-
.../direct/UnboundedReadEvaluatorFactory.java | 22 +-
.../runners/direct/ViewEvaluatorFactory.java | 8 +-
.../beam/runners/direct/WatermarkManager.java | 6 +-
.../runners/direct/WindowEvaluatorFactory.java | 3 +-
.../runners/direct/AggregatorContainerTest.java | 16 +-
.../direct/BoundedReadEvaluatorFactoryTest.java | 5 -
.../CopyOnAccessInMemoryStateInternalsTest.java | 4 +-
.../runners/direct/DirectGraphVisitorTest.java | 16 +-
.../beam/runners/direct/DirectRunnerTest.java | 5 -
...leManagerRemovingTransformEvaluatorTest.java | 103 +-
.../runners/direct/EvaluationContextTest.java | 6 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 3 +-
.../StatefulParDoEvaluatorFactoryTest.java | 4 +-
.../UnboundedReadEvaluatorFactoryTest.java | 13 +-
runners/flink/README.md | 6 +-
runners/flink/examples/pom.xml | 2 +-
.../beam/runners/flink/examples/WordCount.java | 2 +-
.../flink/examples/streaming/AutoComplete.java | 2 +-
.../examples/streaming/KafkaIOExamples.java | 4 +-
.../KafkaWindowedWordCountExample.java | 2 +-
.../examples/streaming/WindowedWordCount.java | 2 +-
runners/flink/pom.xml | 2 +-
runners/flink/runner/pom.xml | 4 +-
.../runners/flink/FlinkPipelineOptions.java | 6 +-
.../runners/flink/OldPerKeyCombineFnRunner.java | 62 +
.../flink/OldPerKeyCombineFnRunners.java | 155 +++
.../FlinkBatchTransformTranslators.java | 40 +-
.../FlinkBatchTranslationContext.java | 21 +-
.../FlinkStreamingTransformTranslators.java | 46 +-
.../FlinkStreamingTranslationContext.java | 20 +-
.../functions/FlinkDoFnFunction.java | 4 +-
.../FlinkMergingNonShuffleReduceFunction.java | 10 +-
.../FlinkMergingPartialReduceFunction.java | 8 +-
.../functions/FlinkMergingReduceFunction.java | 8 +-
.../functions/FlinkMultiOutputDoFnFunction.java | 4 +-
.../FlinkMultiOutputProcessContext.java | 2 +-
.../functions/FlinkNoElementAssignContext.java | 2 +-
.../functions/FlinkPartialReduceFunction.java | 10 +-
.../functions/FlinkProcessContextBase.java | 4 +-
.../functions/FlinkReduceFunction.java | 10 +-
.../FlinkSingleOutputProcessContext.java | 2 +-
.../wrappers/streaming/DoFnOperator.java | 6 +-
.../streaming/SingletonKeyedWorkItemCoder.java | 10 +-
.../wrappers/streaming/WindowDoFnOperator.java | 2 +-
.../streaming/io/BoundedSourceWrapper.java | 2 +-
.../streaming/io/UnboundedFlinkSink.java | 6 +
.../streaming/io/UnboundedSourceWrapper.java | 2 +-
.../beam/runners/flink/PipelineOptionsTest.java | 13 +
.../streaming/FlinkStateInternalsTest.java | 2 +-
.../streaming/UnboundedSourceWrapperTest.java | 464 +++----
runners/google-cloud-dataflow-java/pom.xml | 15 +-
.../beam/runners/dataflow/AssignWindows.java | 89 ++
.../dataflow/DataflowAggregatorTransforms.java | 79 ++
.../dataflow/DataflowMetricUpdateExtractor.java | 109 ++
.../runners/dataflow/DataflowPipelineJob.java | 2 -
.../dataflow/DataflowPipelineTranslator.java | 510 +++-----
.../beam/runners/dataflow/DataflowRunner.java | 109 +-
.../DataflowUnboundedReadFromBoundedSource.java | 547 +++++++++
.../beam/runners/dataflow/ReadTranslator.java | 102 ++
.../runners/dataflow/TransformTranslator.java | 120 ++
.../dataflow/internal/AssignWindows.java | 89 --
.../dataflow/internal/CustomSources.java | 5 -
.../internal/DataflowAggregatorTransforms.java | 79 --
.../internal/DataflowMetricUpdateExtractor.java | 109 --
.../DataflowUnboundedReadFromBoundedSource.java | 556 ---------
.../runners/dataflow/internal/IsmFormat.java | 20 +-
.../dataflow/internal/ReadTranslator.java | 107 --
.../DataflowPipelineWorkerPoolOptions.java | 16 +-
.../beam/runners/dataflow/util/DoFnInfo.java | 66 +-
.../beam/runners/dataflow/util/GcsStager.java | 18 +-
.../beam/runners/dataflow/util/PackageUtil.java | 352 ++++--
.../beam/runners/dataflow/dataflow.properties | 6 +-
.../dataflow/DataflowPipelineJobTest.java | 38 +-
.../DataflowPipelineTranslatorTest.java | 3 +-
.../runners/dataflow/DataflowRunnerTest.java | 8 +-
...aflowUnboundedReadFromBoundedSourceTest.java | 79 ++
...aflowUnboundedReadFromBoundedSourceTest.java | 83 --
.../DataflowPipelineDebugOptionsTest.java | 2 +-
.../options/DataflowPipelineOptionsTest.java | 4 +-
.../options/DataflowProfilingOptionsTest.java | 4 +-
.../runners/dataflow/util/PackageUtilTest.java | 69 +-
runners/pom.xml | 16 +-
runners/spark/README.md | 8 +-
runners/spark/pom.xml | 29 +-
.../spark/aggregators/NamedAggregators.java | 4 +-
.../coders/BeamSparkRunnerRegistrator.java | 48 +-
.../spark/coders/StatelessJavaSerializer.java | 97 ++
.../runners/spark/coders/WritableCoder.java | 4 +-
.../beam/runners/spark/examples/WordCount.java | 2 +-
.../beam/runners/spark/io/MicrobatchSource.java | 9 +-
.../runners/spark/io/SparkUnboundedSource.java | 127 +-
.../spark/stateful/StateSpecFunctions.java | 37 +-
.../runners/spark/translation/DoFnFunction.java | 15 +-
.../spark/translation/EvaluationContext.java | 83 +-
.../translation/GroupCombineFunctions.java | 8 +-
.../spark/translation/MultiDoFnFunction.java | 14 +-
.../translation/SparkAbstractCombineFn.java | 12 +-
.../spark/translation/SparkGlobalCombineFn.java | 13 +-
.../translation/SparkGroupAlsoByWindowFn.java | 2 +-
.../spark/translation/SparkKeyedCombineFn.java | 13 +-
.../spark/translation/SparkPCollectionView.java | 99 ++
.../spark/translation/SparkRuntimeContext.java | 63 +-
.../spark/translation/TransformTranslator.java | 67 +-
.../spark/translation/TranslationUtils.java | 37 +-
.../streaming/StreamingTransformTranslator.java | 114 +-
.../runners/spark/util/BroadcastHelper.java | 127 --
.../runners/spark/util/SideInputBroadcast.java | 77 ++
.../spark/util/SparkSideInputReader.java | 8 +-
.../coders/BeamSparkRunnerRegistratorTest.java | 57 -
.../streaming/KafkaStreamingTest.java | 57 +-
.../ResumeFromCheckpointStreamingTest.java | 20 +-
.../streaming/utils/PAssertStreaming.java | 4 +-
sdks/java/build-tools/pom.xml | 2 +-
.../src/main/resources/beam/findbugs-filter.xml | 26 -
sdks/java/core/pom.xml | 2 +-
.../beam/sdk/annotations/Experimental.java | 5 +-
.../org/apache/beam/sdk/coders/AtomicCoder.java | 2 +-
.../org/apache/beam/sdk/coders/AvroCoder.java | 30 +-
.../apache/beam/sdk/coders/BigDecimalCoder.java | 6 +-
.../beam/sdk/coders/BigEndianIntegerCoder.java | 7 +
.../beam/sdk/coders/BigEndianLongCoder.java | 7 +
.../apache/beam/sdk/coders/ByteArrayCoder.java | 7 +
.../org/apache/beam/sdk/coders/ByteCoder.java | 7 +
.../apache/beam/sdk/coders/ByteStringCoder.java | 8 +
.../java/org/apache/beam/sdk/coders/Coder.java | 7 +
.../apache/beam/sdk/coders/CollectionCoder.java | 12 +-
.../org/apache/beam/sdk/coders/CustomCoder.java | 18 +-
.../apache/beam/sdk/coders/DelegateCoder.java | 29 +-
.../org/apache/beam/sdk/coders/DoubleCoder.java | 7 +
.../apache/beam/sdk/coders/DurationCoder.java | 8 +
.../apache/beam/sdk/coders/InstantCoder.java | 7 +
.../apache/beam/sdk/coders/IterableCoder.java | 12 +-
.../org/apache/beam/sdk/coders/JAXBCoder.java | 48 +-
.../org/apache/beam/sdk/coders/KvCoder.java | 35 +-
.../beam/sdk/coders/LengthPrefixCoder.java | 145 +++
.../org/apache/beam/sdk/coders/ListCoder.java | 7 +
.../org/apache/beam/sdk/coders/MapCoder.java | 62 +-
.../apache/beam/sdk/coders/NullableCoder.java | 6 +
.../beam/sdk/coders/SerializableCoder.java | 17 +-
.../org/apache/beam/sdk/coders/SetCoder.java | 12 +-
.../apache/beam/sdk/coders/StandardCoder.java | 39 +-
.../beam/sdk/coders/StringDelegateCoder.java | 16 +-
.../apache/beam/sdk/coders/StringUtf8Coder.java | 7 +
.../beam/sdk/coders/TableRowJsonCoder.java | 7 +
.../beam/sdk/coders/TextualIntegerCoder.java | 8 +
.../org/apache/beam/sdk/coders/VarIntCoder.java | 10 +-
.../apache/beam/sdk/coders/VarLongCoder.java | 7 +
.../org/apache/beam/sdk/coders/VoidCoder.java | 7 +
.../beam/sdk/coders/protobuf/ProtoCoder.java | 8 +-
.../java/org/apache/beam/sdk/io/AvroSource.java | 5 -
.../sdk/io/BoundedReadFromUnboundedSource.java | 79 +-
.../org/apache/beam/sdk/io/BoundedSource.java | 8 -
.../apache/beam/sdk/io/CompressedSource.java | 8 -
.../org/apache/beam/sdk/io/CountingSource.java | 5 -
.../org/apache/beam/sdk/io/FileSystems.java | 32 +-
.../java/org/apache/beam/sdk/io/PubsubIO.java | 1142 +++++++++---------
.../apache/beam/sdk/io/PubsubUnboundedSink.java | 88 +-
.../beam/sdk/io/PubsubUnboundedSource.java | 104 +-
.../main/java/org/apache/beam/sdk/io/Read.java | 7 +-
.../java/org/apache/beam/sdk/io/TextIO.java | 5 -
.../java/org/apache/beam/sdk/io/XmlSource.java | 5 -
.../org/apache/beam/sdk/options/GcpOptions.java | 36 +-
.../org/apache/beam/sdk/options/GcsOptions.java | 4 +-
.../beam/sdk/options/PipelineOptions.java | 2 +-
.../sdk/options/PipelineOptionsFactory.java | 10 +-
.../apache/beam/sdk/options/ValueProvider.java | 6 +-
.../beam/sdk/runners/TransformHierarchy.java | 33 +-
.../testing/FlattenWithHeterogeneousCoders.java | 29 +
.../org/apache/beam/sdk/testing/PAssert.java | 12 +-
.../beam/sdk/testing/RunnableOnService.java | 14 +-
.../beam/sdk/testing/SourceTestUtils.java | 5 -
.../org/apache/beam/sdk/testing/TestStream.java | 8 +
.../sdk/testing/UsesUnboundedPCollections.java | 23 +
.../beam/sdk/testing/ValueInSingleWindow.java | 6 +-
.../sdk/transforms/AggregatorRetriever.java | 13 +-
.../beam/sdk/transforms/AppliedPTransform.java | 11 +-
.../org/apache/beam/sdk/transforms/Combine.java | 197 +--
.../apache/beam/sdk/transforms/CombineFns.java | 14 +-
.../org/apache/beam/sdk/transforms/Count.java | 4 +-
.../org/apache/beam/sdk/transforms/Create.java | 5 -
.../sdk/transforms/DelegatingAggregator.java | 2 +-
.../beam/sdk/transforms/DoFnAdapters.java | 504 --------
.../apache/beam/sdk/transforms/DoFnTester.java | 7 -
.../apache/beam/sdk/transforms/GroupByKey.java | 2 +-
.../org/apache/beam/sdk/transforms/Max.java | 124 +-
.../org/apache/beam/sdk/transforms/Mean.java | 27 +-
.../org/apache/beam/sdk/transforms/Min.java | 122 +-
.../org/apache/beam/sdk/transforms/OldDoFn.java | 758 ------------
.../apache/beam/sdk/transforms/PTransform.java | 9 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 46 +-
.../org/apache/beam/sdk/transforms/Regex.java | 589 ++++++++-
.../org/apache/beam/sdk/transforms/Sum.java | 57 +-
.../apache/beam/sdk/transforms/ToString.java | 198 +++
.../org/apache/beam/sdk/transforms/Top.java | 27 +-
.../beam/sdk/transforms/join/CoGbkResult.java | 35 +-
.../sdk/transforms/reflect/DoFnInvoker.java | 20 -
.../sdk/transforms/reflect/DoFnInvokers.java | 142 +--
.../sdk/transforms/reflect/DoFnSignature.java | 15 +-
.../windowing/AfterDelayFromFirstElement.java | 2 +-
.../sdk/transforms/windowing/AfterPane.java | 2 +-
.../sdk/transforms/windowing/GlobalWindow.java | 6 +
.../transforms/windowing/IntervalWindow.java | 4 +-
.../beam/sdk/transforms/windowing/Window.java | 3 +-
.../org/apache/beam/sdk/util/CoderUtils.java | 28 +-
.../beam/sdk/util/CombineContextFactory.java | 18 -
.../org/apache/beam/sdk/util/DefaultBucket.java | 105 ++
.../util/EmptyOnDeserializationThreadLocal.java | 39 +
.../apache/beam/sdk/util/GcpProjectUtil.java | 2 +-
.../java/org/apache/beam/sdk/util/GcsUtil.java | 36 +-
.../org/apache/beam/sdk/util/NameUtils.java | 162 +++
.../org/apache/beam/sdk/util/PropertyNames.java | 1 +
.../org/apache/beam/sdk/util/PubsubClient.java | 28 +-
.../apache/beam/sdk/util/PubsubGrpcClient.java | 6 +-
.../apache/beam/sdk/util/PubsubJsonClient.java | 4 +-
.../apache/beam/sdk/util/PubsubTestClient.java | 6 +-
.../org/apache/beam/sdk/util/StringUtils.java | 100 --
.../apache/beam/sdk/util/TimerInternals.java | 4 +-
.../org/apache/beam/sdk/util/WindowedValue.java | 23 +-
.../beam/sdk/util/state/StateContexts.java | 4 +-
.../org/apache/beam/sdk/values/PValueBase.java | 4 +-
.../beam/sdk/values/TimestampedValue.java | 10 +-
.../sdk/AggregatorPipelineExtractorTest.java | 16 +-
.../apache/beam/sdk/coders/AvroCoderTest.java | 7 +
.../beam/sdk/coders/BigDecimalCoderTest.java | 46 +-
.../sdk/coders/BigEndianIntegerCoderTest.java | 9 +
.../beam/sdk/coders/BigEndianLongCoderTest.java | 9 +
.../beam/sdk/coders/ByteArrayCoderTest.java | 6 +
.../apache/beam/sdk/coders/ByteCoderTest.java | 9 +
.../beam/sdk/coders/ByteStringCoderTest.java | 8 +
.../beam/sdk/coders/CoderRegistryTest.java | 6 +
.../org/apache/beam/sdk/coders/CoderTest.java | 8 +
.../beam/sdk/coders/CollectionCoderTest.java | 16 +
.../beam/sdk/coders/DefaultCoderTest.java | 4 +-
.../beam/sdk/coders/DelegateCoderTest.java | 35 +-
.../apache/beam/sdk/coders/DoubleCoderTest.java | 9 +
.../beam/sdk/coders/DurationCoderTest.java | 10 +
.../beam/sdk/coders/InstantCoderTest.java | 9 +
.../beam/sdk/coders/IterableCoderTest.java | 27 +-
.../apache/beam/sdk/coders/JAXBCoderTest.java | 26 +-
.../org/apache/beam/sdk/coders/KvCoderTest.java | 29 +
.../beam/sdk/coders/LengthPrefixCoderTest.java | 129 ++
.../apache/beam/sdk/coders/ListCoderTest.java | 16 +-
.../apache/beam/sdk/coders/MapCoderTest.java | 21 +-
.../beam/sdk/coders/NullableCoderTest.java | 12 +
.../beam/sdk/coders/SerializableCoderTest.java | 9 +
.../apache/beam/sdk/coders/SetCoderTest.java | 16 +
.../beam/sdk/coders/StandardCoderTest.java | 40 +
.../sdk/coders/StringDelegateCoderTest.java | 11 +
.../beam/sdk/coders/StringUtf8CoderTest.java | 9 +
.../beam/sdk/coders/TableRowJsonCoderTest.java | 9 +
.../sdk/coders/TextualIntegerCoderTest.java | 9 +
.../apache/beam/sdk/coders/VarIntCoderTest.java | 9 +
.../beam/sdk/coders/VarLongCoderTest.java | 9 +
.../apache/beam/sdk/coders/VoidCoderTest.java | 40 +
.../beam/sdk/io/AvroIOGeneratedClassTest.java | 285 -----
.../apache/beam/sdk/io/AvroIOTransformTest.java | 324 +++++
.../beam/sdk/io/CompressedSourceTest.java | 5 -
.../apache/beam/sdk/io/FileBasedSourceTest.java | 5 -
.../org/apache/beam/sdk/io/FileSystemsTest.java | 33 +-
.../beam/sdk/io/OffsetBasedSourceTest.java | 5 -
.../org/apache/beam/sdk/io/PubsubIOTest.java | 86 +-
.../beam/sdk/io/PubsubUnboundedSinkTest.java | 41 +-
.../beam/sdk/io/PubsubUnboundedSourceTest.java | 10 +-
.../java/org/apache/beam/sdk/io/ReadTest.java | 5 -
.../java/org/apache/beam/sdk/io/WriteTest.java | 10 +-
.../apache/beam/sdk/options/GcpOptionsTest.java | 4 +-
.../sdk/options/PipelineOptionsFactoryTest.java | 6 +-
.../beam/sdk/options/PipelineOptionsTest.java | 3 +-
.../beam/sdk/options/ValueProviderTest.java | 36 +-
.../sdk/options/ValueProviderUtilsTest.java | 2 +-
.../sdk/runners/TransformHierarchyTest.java | 30 +-
.../apache/beam/sdk/testing/TestStreamTest.java | 5 +
.../testing/ValueInSingleWindowCoderTest.java | 7 +
.../sdk/transforms/ApproximateUniqueTest.java | 483 ++++----
.../beam/sdk/transforms/CombineFnsTest.java | 20 +-
.../apache/beam/sdk/transforms/CombineTest.java | 99 +-
.../apache/beam/sdk/transforms/CountTest.java | 2 +-
.../apache/beam/sdk/transforms/CreateTest.java | 8 -
.../DoFnDelegatingAggregatorTest.java | 142 ---
.../apache/beam/sdk/transforms/DoFnTest.java | 15 +-
.../beam/sdk/transforms/DoFnTesterTest.java | 6 +-
.../apache/beam/sdk/transforms/FlattenTest.java | 27 +
.../apache/beam/sdk/transforms/KvSwapTest.java | 13 +-
.../org/apache/beam/sdk/transforms/MaxTest.java | 20 +-
.../apache/beam/sdk/transforms/MeanTest.java | 7 +-
.../org/apache/beam/sdk/transforms/MinTest.java | 21 +-
.../apache/beam/sdk/transforms/NoOpOldDoFn.java | 71 --
.../beam/sdk/transforms/OldDoFnContextTest.java | 69 --
.../apache/beam/sdk/transforms/OldDoFnTest.java | 188 ---
.../apache/beam/sdk/transforms/ParDoTest.java | 74 +-
.../apache/beam/sdk/transforms/RegexTest.java | 127 +-
.../apache/beam/sdk/transforms/SampleTest.java | 405 ++++---
.../beam/sdk/transforms/SimpleStatsFnsTest.java | 36 +-
.../org/apache/beam/sdk/transforms/SumTest.java | 24 +-
.../beam/sdk/transforms/ToStringTest.java | 125 ++
.../org/apache/beam/sdk/transforms/TopTest.java | 13 +-
.../apache/beam/sdk/transforms/ViewTest.java | 2 +-
.../transforms/join/CoGbkResultCoderTest.java | 10 +-
.../sdk/transforms/join/UnionCoderTest.java | 24 +-
.../transforms/reflect/DoFnInvokersTest.java | 42 -
.../transforms/windowing/GlobalWindowTest.java | 64 +
.../apache/beam/sdk/util/CombineFnUtilTest.java | 8 +-
.../apache/beam/sdk/util/DefaultBucketTest.java | 112 ++
.../org/apache/beam/sdk/util/GcsUtilTest.java | 56 +
.../org/apache/beam/sdk/util/NameUtilsTest.java | 177 +++
.../beam/sdk/util/PubsubGrpcClientTest.java | 8 +-
.../beam/sdk/util/PubsubJsonClientTest.java | 3 +-
.../beam/sdk/util/PubsubTestClientTest.java | 4 +-
.../beam/sdk/util/SerializableUtilsTest.java | 4 +-
.../apache/beam/sdk/util/StringUtilsTest.java | 100 --
.../beam/sdk/util/TimerInternalsTest.java | 5 +
.../beam/sdk/util/ValueWithRecordIdTest.java | 34 +
.../apache/beam/sdk/util/WindowedValueTest.java | 23 +
.../util/state/InMemoryStateInternalsTest.java | 2 +-
.../beam/sdk/util/state/StateTagTest.java | 11 +-
.../beam/sdk/values/TimestampedValueTest.java | 19 +-
sdks/java/extensions/join-library/README.md | 10 -
sdks/java/extensions/join-library/pom.xml | 2 +-
sdks/java/extensions/pom.xml | 2 +-
sdks/java/extensions/sorter/pom.xml | 2 +-
sdks/java/io/elasticsearch/pom.xml | 175 +++
.../sdk/io/elasticsearch/ElasticsearchIO.java | 819 +++++++++++++
.../beam/sdk/io/elasticsearch/package-info.java | 20 +
.../elasticsearch/ElasticSearchIOTestUtils.java | 129 ++
.../io/elasticsearch/ElasticsearchIOTest.java | 358 ++++++
sdks/java/io/google-cloud-platform/pom.xml | 2 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 308 +++--
.../sdk/io/gcp/bigquery/BigQueryServices.java | 16 +-
.../io/gcp/bigquery/BigQueryServicesImpl.java | 76 +-
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 8 -
.../io/gcp/bigtable/BigtableTestOptions.java | 37 -
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 205 +++-
.../gcp/bigquery/BigQueryServicesImplTest.java | 141 +++
.../sdk/io/gcp/bigquery/BigQueryUtilTest.java | 3 +-
.../sdk/io/gcp/bigtable/BigtableIOTest.java | 5 +-
.../io/gcp/bigtable/BigtableTestOptions.java | 37 +
sdks/java/io/hdfs/pom.xml | 2 +-
.../beam/sdk/io/hdfs/AvroWrapperCoder.java | 4 +-
.../apache/beam/sdk/io/hdfs/HDFSFileSource.java | 5 -
.../apache/beam/sdk/io/hdfs/WritableCoder.java | 4 +-
.../beam/sdk/io/hdfs/AvroWrapperCoderTest.java | 1 -
sdks/java/io/jdbc/pom.xml | 2 +-
sdks/java/io/jms/pom.xml | 2 +-
sdks/java/io/kafka/pom.xml | 2 +-
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 7 +-
.../beam/sdk/io/kafka/KafkaRecordCoder.java | 4 +-
.../apache/beam/sdk/io/kafka/KafkaIOTest.java | 9 +-
.../beam/sdk/io/kafka/KafkaRecordCoderTest.java | 34 +
sdks/java/io/kinesis/pom.xml | 2 +-
.../beam/sdk/io/kinesis/KinesisRecordCoder.java | 4 +-
.../beam/sdk/io/kinesis/package-info.java | 2 +-
sdks/java/io/mongodb/pom.xml | 2 +-
.../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 5 -
.../apache/beam/sdk/io/mongodb/MongoDbIO.java | 5 -
sdks/java/io/mqtt/pom.xml | 152 +++
.../org/apache/beam/sdk/io/mqtt/MqttIO.java | 588 +++++++++
.../apache/beam/sdk/io/mqtt/package-info.java | 22 +
.../org/apache/beam/sdk/io/mqtt/MqttIOTest.java | 197 +++
sdks/java/io/pom.xml | 4 +-
sdks/java/java8tests/pom.xml | 2 +-
.../maven-archetypes/examples-java8/pom.xml | 2 +-
.../main/resources/archetype-resources/pom.xml | 19 +-
sdks/java/maven-archetypes/examples/pom.xml | 2 +-
.../main/resources/archetype-resources/pom.xml | 19 +-
sdks/java/maven-archetypes/pom.xml | 2 +-
sdks/java/maven-archetypes/starter/pom.xml | 2 +-
.../main/resources/archetype-resources/pom.xml | 4 +-
.../resources/projects/basic/reference/pom.xml | 4 +-
sdks/java/pom.xml | 2 +-
sdks/pom.xml | 15 +-
sdks/python/pom.xml | 2 +-
466 files changed, 13883 insertions(+), 8047 deletions(-)
----------------------------------------------------------------------