You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ro...@apache.org on 2016/10/27 17:28:36 UTC
[48/50] [abbrv] incubator-beam git commit: Merge branch 'master' into
py_merge_master
Merge branch 'master' into py_merge_master
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/02b5b4c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/02b5b4c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/02b5b4c9
Branch: refs/heads/python-sdk
Commit: 02b5b4c9ddd9a57e31a5d0e3941abe9e5c4fe9ee
Parents: c37b89e eba099f
Author: Vikas Kedigehalli <vi...@google.com>
Authored: Wed Oct 26 15:43:58 2016 -0700
Committer: Vikas Kedigehalli <vi...@google.com>
Committed: Wed Oct 26 15:43:58 2016 -0700
----------------------------------------------------------------------
.gitignore | 8 +
.travis.yml | 35 +-
.travis/README.md | 23 +
.travis/settings.xml | 33 +
.travis/test_wordcount.sh | 125 ++
README.md | 3 +-
examples/java/pom.xml | 280 +++-
.../beam/examples/DebuggingWordCount.java | 32 +-
.../apache/beam/examples/MinimalWordCount.java | 12 +-
.../apache/beam/examples/WindowedWordCount.java | 46 +-
.../org/apache/beam/examples/WordCount.java | 39 +-
.../common/ExampleBigQueryTableOptions.java | 8 +-
.../beam/examples/common/ExampleOptions.java | 48 +-
...xamplePubsubTopicAndSubscriptionOptions.java | 4 +-
.../common/ExamplePubsubTopicOptions.java | 4 +-
.../beam/examples/common/ExampleUtils.java | 95 +-
.../examples/common/PubsubFileInjector.java | 155 --
.../beam/examples/complete/AutoComplete.java | 95 +-
.../org/apache/beam/examples/complete/README.md | 4 +-
.../examples/complete/StreamingWordExtract.java | 20 +-
.../apache/beam/examples/complete/TfIdf.java | 44 +-
.../examples/complete/TopWikipediaSessions.java | 28 +-
.../examples/complete/TrafficMaxLaneFlow.java | 29 +-
.../beam/examples/complete/TrafficRoutes.java | 39 +-
.../examples/cookbook/BigQueryTornadoes.java | 32 +-
.../cookbook/CombinePerKeyExamples.java | 32 +-
.../examples/cookbook/DatastoreWordCount.java | 266 ----
.../beam/examples/cookbook/DeDupExample.java | 24 +-
.../beam/examples/cookbook/FilterExamples.java | 37 +-
.../beam/examples/cookbook/JoinExamples.java | 23 +-
.../examples/cookbook/MaxPerKeyExamples.java | 33 +-
.../beam/examples/cookbook/TriggerExample.java | 59 +-
.../beam/examples/DebuggingWordCountTest.java | 6 +-
.../beam/examples/WindowedWordCountIT.java | 75 +
.../org/apache/beam/examples/WordCountIT.java | 125 +-
.../org/apache/beam/examples/WordCountTest.java | 6 +-
.../examples/complete/AutoCompleteTest.java | 12 +-
.../beam/examples/complete/TfIdfTest.java | 6 +-
.../complete/TopWikipediaSessionsTest.java | 7 +-
.../examples/cookbook/BigQueryTornadoesIT.java | 15 +-
.../cookbook/BigQueryTornadoesTest.java | 7 +-
.../cookbook/CombinePerKeyExamplesTest.java | 7 +-
.../examples/cookbook/DeDupExampleTest.java | 6 +-
.../examples/cookbook/FilterExamplesTest.java | 9 +-
.../examples/cookbook/JoinExamplesTest.java | 9 +-
.../cookbook/MaxPerKeyExamplesTest.java | 9 +-
.../examples/cookbook/TriggerExampleTest.java | 17 +-
examples/java/src/test/resources/LICENSE | 202 +++
examples/java8/pom.xml | 48 +-
.../beam/examples/MinimalWordCountJava8.java | 5 +-
.../beam/examples/complete/game/GameStats.java | 70 +-
.../examples/complete/game/HourlyTeamScore.java | 46 +-
.../examples/complete/game/LeaderBoard.java | 176 ++-
.../beam/examples/complete/game/UserScore.java | 34 +-
.../complete/game/injector/Injector.java | 18 +-
.../complete/game/injector/InjectorUtils.java | 1 -
.../injector/RetryHttpInitializerWrapper.java | 1 -
.../complete/game/utils/WriteToBigQuery.java | 67 +-
.../game/utils/WriteWindowedToBigQuery.java | 20 +-
.../examples/MinimalWordCountJava8Test.java | 23 +-
.../examples/complete/game/GameStatsTest.java | 8 +-
.../complete/game/HourlyTeamScoreTest.java | 8 +-
.../examples/complete/game/LeaderBoardTest.java | 361 +++++
.../examples/complete/game/UserScoreTest.java | 8 +-
examples/pom.xml | 2 +-
pom.xml | 167 +-
runners/core-java/pom.xml | 8 +-
.../beam/runners/core/AggregatorFactory.java | 39 +
.../apache/beam/runners/core/AssignWindows.java | 46 +
.../beam/runners/core/AssignWindowsDoFn.java | 79 +
.../apache/beam/runners/core/DoFnRunner.java | 65 +
.../apache/beam/runners/core/DoFnRunners.java | 257 +++
.../runners/core/ElementAndRestriction.java | 42 +
.../core/ElementAndRestrictionCoder.java | 67 +
.../runners/core/ElementByteSizeObservable.java | 43 +
.../runners/core/GBKIntoKeyedWorkItems.java | 55 +
.../core/GroupAlsoByWindowViaWindowSetDoFn.java | 10 +-
.../runners/core/GroupAlsoByWindowsDoFn.java | 47 +
.../GroupAlsoByWindowsViaOutputBufferDoFn.java | 110 ++
.../core/GroupByKeyViaGroupByKeyOnly.java | 271 ++++
.../core/LateDataDroppingDoFnRunner.java | 151 ++
.../apache/beam/runners/core/NonEmptyPanes.java | 151 ++
.../beam/runners/core/PaneInfoTracker.java | 158 ++
.../beam/runners/core/PeekingReiterator.java | 100 ++
.../core/PushbackSideInputDoFnRunner.java | 116 ++
.../org/apache/beam/runners/core/ReduceFn.java | 130 ++
.../runners/core/ReduceFnContextFactory.java | 499 ++++++
.../beam/runners/core/ReduceFnRunner.java | 997 ++++++++++++
.../beam/runners/core/SideInputHandler.java | 243 +++
.../beam/runners/core/SimpleDoFnRunner.java | 586 +++++++
.../beam/runners/core/SimpleOldDoFnRunner.java | 521 ++++++
.../beam/runners/core/SplittableParDo.java | 469 ++++++
.../beam/runners/core/SystemReduceFn.java | 139 ++
.../core/UnboundedReadFromBoundedSource.java | 45 +-
.../apache/beam/runners/core/WatermarkHold.java | 539 +++++++
.../core/triggers/AfterAllStateMachine.java | 109 ++
.../AfterDelayFromFirstElementStateMachine.java | 337 ++++
.../core/triggers/AfterEachStateMachine.java | 130 ++
.../core/triggers/AfterFirstStateMachine.java | 112 ++
.../core/triggers/AfterPaneStateMachine.java | 139 ++
.../AfterProcessingTimeStateMachine.java | 93 ++
...rSynchronizedProcessingTimeStateMachine.java | 63 +
.../triggers/AfterWatermarkStateMachine.java | 325 ++++
.../triggers/DefaultTriggerStateMachine.java | 81 +
.../triggers/ExecutableTriggerStateMachine.java | 160 ++
.../runners/core/triggers/FinishedTriggers.java | 44 +
.../core/triggers/FinishedTriggersBitSet.java | 67 +
.../core/triggers/FinishedTriggersSet.java | 72 +
.../core/triggers/NeverStateMachine.java | 60 +
.../core/triggers/OrFinallyStateMachine.java | 85 +
.../core/triggers/RepeatedlyStateMachine.java | 88 ++
.../triggers/ReshuffleTriggerStateMachine.java | 50 +
.../core/triggers/TriggerStateMachine.java | 487 ++++++
.../TriggerStateMachineContextFactory.java | 509 ++++++
.../triggers/TriggerStateMachineRunner.java | 234 +++
.../core/triggers/TriggerStateMachines.java | 215 +++
.../runners/core/triggers/package-info.java | 23 +
.../org/apache/beam/sdk/util/AssignWindows.java | 46 -
.../apache/beam/sdk/util/AssignWindowsDoFn.java | 80 -
.../beam/sdk/util/BatchTimerInternals.java | 141 --
.../org/apache/beam/sdk/util/DoFnRunner.java | 63 -
.../apache/beam/sdk/util/DoFnRunnerBase.java | 551 -------
.../org/apache/beam/sdk/util/DoFnRunners.java | 222 ---
.../beam/sdk/util/GroupAlsoByWindowsDoFn.java | 63 -
.../GroupAlsoByWindowsViaOutputBufferDoFn.java | 100 --
.../sdk/util/GroupByKeyViaGroupByKeyOnly.java | 269 ----
.../sdk/util/LateDataDroppingDoFnRunner.java | 147 --
.../org/apache/beam/sdk/util/NonEmptyPanes.java | 150 --
.../apache/beam/sdk/util/PaneInfoTracker.java | 158 --
.../sdk/util/PushbackSideInputDoFnRunner.java | 115 --
.../java/org/apache/beam/sdk/util/ReduceFn.java | 130 --
.../beam/sdk/util/ReduceFnContextFactory.java | 497 ------
.../apache/beam/sdk/util/ReduceFnRunner.java | 988 ------------
.../apache/beam/sdk/util/SimpleDoFnRunner.java | 55 -
.../apache/beam/sdk/util/SystemReduceFn.java | 139 --
.../org/apache/beam/sdk/util/TriggerRunner.java | 245 ---
.../org/apache/beam/sdk/util/WatermarkHold.java | 540 -------
.../util/common/ElementByteSizeObservable.java | 42 -
.../beam/sdk/util/common/PeekingReiterator.java | 99 --
.../beam/sdk/util/common/package-info.java | 20 -
.../org/apache/beam/sdk/util/package-info.java | 20 -
.../core/ElementAndRestrictionCoderTest.java | 127 ++
.../core/GroupAlsoByWindowsProperties.java | 660 ++++++++
...oupAlsoByWindowsViaOutputBufferDoFnTest.java | 110 ++
.../core/LateDataDroppingDoFnRunnerTest.java | 117 ++
.../core/PushbackSideInputDoFnRunnerTest.java | 235 +++
.../beam/runners/core/ReduceFnRunnerTest.java | 1491 ++++++++++++++++++
.../beam/runners/core/ReduceFnTester.java | 659 ++++++++
.../beam/runners/core/SideInputHandlerTest.java | 220 +++
.../runners/core/SimpleOldDoFnRunnerTest.java | 88 ++
.../beam/runners/core/SplittableParDoTest.java | 467 ++++++
.../UnboundedReadFromBoundedSourceTest.java | 27 +-
.../core/triggers/AfterAllStateMachineTest.java | 140 ++
.../triggers/AfterEachStateMachineTest.java | 108 ++
.../triggers/AfterFirstStateMachineTest.java | 159 ++
.../triggers/AfterPaneStateMachineTest.java | 117 ++
.../AfterProcessingTimeStateMachineTest.java | 172 ++
...chronizedProcessingTimeStateMachineTest.java | 110 ++
.../AfterWatermarkStateMachineTest.java | 382 +++++
.../DefaultTriggerStateMachineTest.java | 165 ++
.../ExecutableTriggerStateMachineTest.java | 108 ++
.../triggers/FinishedTriggersBitSetTest.java | 55 +
.../triggers/FinishedTriggersProperties.java | 115 ++
.../core/triggers/FinishedTriggersSetTest.java | 60 +
.../core/triggers/NeverStateMachineTest.java | 59 +
.../triggers/OrFinallyStateMachineTest.java | 177 +++
.../triggers/RepeatedlyStateMachineTest.java | 200 +++
.../ReshuffleTriggerStateMachineTest.java | 68 +
.../core/triggers/StubTriggerStateMachine.java | 60 +
.../core/triggers/TriggerStateMachineTest.java | 98 ++
.../triggers/TriggerStateMachineTester.java | 431 +++++
.../core/triggers/TriggerStateMachinesTest.java | 199 +++
.../beam/sdk/util/BatchTimerInternalsTest.java | 118 --
.../sdk/util/GroupAlsoByWindowsProperties.java | 661 --------
...oupAlsoByWindowsViaOutputBufferDoFnTest.java | 110 --
.../util/LateDataDroppingDoFnRunnerTest.java | 117 --
.../util/PushbackSideInputDoFnRunnerTest.java | 234 ---
.../beam/sdk/util/ReduceFnRunnerTest.java | 1447 -----------------
.../apache/beam/sdk/util/ReduceFnTester.java | 789 ---------
.../beam/sdk/util/SimpleDoFnRunnerTest.java | 86 -
runners/direct-java/pom.xml | 32 +-
.../runners/direct/AggregatorContainer.java | 17 +-
.../direct/BoundedReadEvaluatorFactory.java | 159 +-
.../beam/runners/direct/BundleFactory.java | 15 +-
.../runners/direct/CloningBundleFactory.java | 98 ++
.../beam/runners/direct/CloningThreadLocal.java | 43 -
.../beam/runners/direct/CommittedResult.java | 9 +-
.../beam/runners/direct/CompletionCallback.java | 4 +-
.../direct/ConsumerTrackingPipelineVisitor.java | 13 +-
.../runners/direct/DirectExecutionContext.java | 2 +-
...ectGBKIntoKeyedWorkItemsOverrideFactory.java | 66 +
.../beam/runners/direct/DirectMetrics.java | 338 ++++
.../beam/runners/direct/DirectOptions.java | 85 +-
.../beam/runners/direct/DirectRegistrar.java | 5 +-
.../beam/runners/direct/DirectRunner.java | 186 ++-
.../runners/direct/DirectTimerInternals.java | 6 +-
.../runners/direct/DoFnLifecycleManager.java | 117 ++
...ecycleManagerRemovingTransformEvaluator.java | 78 +
.../runners/direct/DoFnLifecycleManagers.java | 45 +
.../beam/runners/direct/EmptyInputProvider.java | 45 +
.../direct/EncodabilityEnforcementFactory.java | 50 +-
.../beam/runners/direct/EvaluationContext.java | 63 +-
.../direct/ExecutorServiceParallelExecutor.java | 167 +-
.../FixedThreadPoolExecutorServiceFactory.java | 45 -
.../runners/direct/FlattenEvaluatorFactory.java | 29 +-
.../runners/direct/ForwardingPTransform.java | 2 +-
.../GroupAlsoByWindowEvaluatorFactory.java | 34 +-
.../direct/GroupByKeyOnlyEvaluatorFactory.java | 42 +-
.../ImmutabilityCheckingBundleFactory.java | 28 +-
.../direct/ImmutabilityEnforcementFactory.java | 5 +-
.../direct/ImmutableListBundleFactory.java | 80 +-
.../direct/KeyedPValueTrackingVisitor.java | 5 +-
.../beam/runners/direct/NanosOffsetClock.java | 16 +-
.../beam/runners/direct/ParDoEvaluator.java | 35 +-
.../direct/ParDoMultiEvaluatorFactory.java | 79 +-
.../runners/direct/ParDoOverrideFactory.java | 55 +
.../direct/ParDoSingleEvaluatorFactory.java | 62 +-
.../beam/runners/direct/PipelineExecutor.java | 5 +-
.../beam/runners/direct/RootInputProvider.java | 46 +
.../runners/direct/RootProviderRegistry.java | 66 +
.../beam/runners/direct/SideInputContainer.java | 21 +-
.../apache/beam/runners/direct/StepAndKey.java | 4 +-
.../runners/direct/StepTransformResult.java | 63 +-
.../beam/runners/direct/StructuralKey.java | 88 +-
.../direct/TestStreamEvaluatorFactory.java | 232 +++
...readLocalInvalidatingTransformEvaluator.java | 63 -
.../beam/runners/direct/TransformEvaluator.java | 2 +-
.../direct/TransformEvaluatorFactory.java | 26 +-
.../direct/TransformEvaluatorRegistry.java | 82 +-
.../beam/runners/direct/TransformExecutor.java | 61 +-
.../direct/TransformExecutorServices.java | 1 -
.../beam/runners/direct/TransformResult.java | 23 +-
.../direct/UnboundedReadDeduplicator.java | 11 +-
.../direct/UnboundedReadEvaluatorFactory.java | 331 ++--
.../direct/UncommittedBundleOutputManager.java | 9 +-
.../runners/direct/ViewEvaluatorFactory.java | 27 +-
.../direct/WatermarkCallbackExecutor.java | 11 +-
.../beam/runners/direct/WatermarkManager.java | 96 +-
.../runners/direct/WindowEvaluatorFactory.java | 31 +-
.../direct/WriteWithShardingFactory.java | 19 +-
.../runners/direct/AggregatorContainerTest.java | 9 +-
.../direct/BoundedReadEvaluatorFactoryTest.java | 194 ++-
.../direct/CloningBundleFactoryTest.java | 177 +++
.../runners/direct/CloningThreadLocalTest.java | 92 --
.../runners/direct/CommittedResultTest.java | 23 +-
.../ConsumerTrackingPipelineVisitorTest.java | 38 +-
.../beam/runners/direct/DirectMetricsTest.java | 133 ++
.../runners/direct/DirectRegistrarTest.java | 9 +-
.../beam/runners/direct/DirectRunnerTest.java | 207 ++-
.../direct/DirectTimerInternalsTest.java | 1 -
...leManagerRemovingTransformEvaluatorTest.java | 142 ++
.../direct/DoFnLifecycleManagerTest.java | 235 +++
.../direct/DoFnLifecycleManagersTest.java | 171 ++
.../EncodabilityEnforcementFactoryTest.java | 142 +-
.../runners/direct/EvaluationContextTest.java | 49 +-
.../direct/FlattenEvaluatorFactoryTest.java | 37 +-
.../direct/ForwardingPTransformTest.java | 8 +-
.../direct/GroupByKeyEvaluatorFactoryTest.java | 35 +-
.../GroupByKeyOnlyEvaluatorFactoryTest.java | 43 +-
.../ImmutabilityCheckingBundleFactoryTest.java | 68 +-
.../ImmutabilityEnforcementFactoryTest.java | 20 +-
.../direct/ImmutableListBundleFactoryTest.java | 63 +-
.../direct/KeyedPValueTrackingVisitorTest.java | 17 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 29 +-
.../direct/ParDoMultiEvaluatorFactoryTest.java | 164 +-
.../direct/ParDoSingleEvaluatorFactoryTest.java | 147 +-
.../runners/direct/SideInputContainerTest.java | 17 +-
.../beam/runners/direct/SplittableDoFnTest.java | 231 +++
.../runners/direct/StepTransformResultTest.java | 5 +-
.../beam/runners/direct/StructuralKeyTest.java | 10 +-
.../direct/TestStreamEvaluatorFactoryTest.java | 171 ++
...LocalInvalidatingTransformEvaluatorTest.java | 135 --
.../direct/TransformExecutorServicesTest.java | 4 +-
.../runners/direct/TransformExecutorTest.java | 138 +-
.../direct/UnboundedReadDeduplicatorTest.java | 23 +-
.../UnboundedReadEvaluatorFactoryTest.java | 345 ++--
.../direct/ViewEvaluatorFactoryTest.java | 10 +-
.../direct/WatermarkCallbackExecutorTest.java | 8 +-
.../runners/direct/WatermarkManagerTest.java | 105 +-
.../direct/WindowEvaluatorFactoryTest.java | 26 +-
.../direct/WriteWithShardingFactoryTest.java | 30 +-
runners/flink/README.md | 27 +-
runners/flink/examples/pom.xml | 4 +-
.../beam/runners/flink/examples/TFIDF.java | 60 +-
.../beam/runners/flink/examples/WordCount.java | 19 +-
.../runners/flink/examples/package-info.java | 22 +
.../flink/examples/streaming/AutoComplete.java | 50 +-
.../flink/examples/streaming/JoinExamples.java | 18 +-
.../examples/streaming/KafkaIOExamples.java | 51 +-
.../KafkaWindowedWordCountExample.java | 41 +-
.../examples/streaming/WindowedWordCount.java | 33 +-
.../flink/examples/streaming/package-info.java | 22 +
runners/flink/pom.xml | 4 +-
runners/flink/runner/pom.xml | 23 +-
.../flink/DefaultParallelismFactory.java | 1 -
.../flink/FlinkDetachedRunnerResult.java | 76 +
.../FlinkPipelineExecutionEnvironment.java | 28 +-
.../runners/flink/FlinkPipelineOptions.java | 82 +-
.../apache/beam/runners/flink/FlinkRunner.java | 504 +++++-
.../runners/flink/FlinkRunnerRegistrar.java | 15 +-
.../beam/runners/flink/FlinkRunnerResult.java | 46 +-
.../beam/runners/flink/TestFlinkRunner.java | 34 +-
.../apache/beam/runners/flink/package-info.java | 22 +
.../FlinkBatchPipelineTranslator.java | 18 +-
.../FlinkBatchTransformTranslators.java | 19 +-
.../FlinkBatchTranslationContext.java | 16 +-
.../translation/FlinkPipelineTranslator.java | 2 +-
.../FlinkStreamingPipelineTranslator.java | 62 +-
.../FlinkStreamingTransformTranslators.java | 972 +++++++++---
.../FlinkStreamingTranslationContext.java | 6 +-
.../flink/translation/TranslationMode.java | 8 +-
.../functions/FlinkAssignContext.java | 6 +-
.../functions/FlinkAssignWindows.java | 4 +-
.../functions/FlinkDoFnFunction.java | 22 +-
.../FlinkMergingNonShuffleReduceFunction.java | 17 +-
.../FlinkMergingPartialReduceFunction.java | 17 +-
.../functions/FlinkMergingReduceFunction.java | 21 +-
.../functions/FlinkMultiOutputDoFnFunction.java | 44 +-
.../FlinkMultiOutputProcessContext.java | 6 +-
.../FlinkMultiOutputPruningFunction.java | 1 -
.../functions/FlinkNoElementAssignContext.java | 1 -
.../functions/FlinkPartialReduceFunction.java | 17 +-
.../functions/FlinkProcessContext.java | 27 +-
.../functions/FlinkReduceFunction.java | 21 +-
.../functions/SideInputInitializer.java | 12 +-
.../translation/functions/package-info.java | 22 +
.../runners/flink/translation/package-info.java | 22 +
.../translation/types/CoderTypeInformation.java | 5 +-
.../translation/types/CoderTypeSerializer.java | 10 +-
.../types/EncodedValueComparator.java | 6 +-
.../types/EncodedValueSerializer.java | 164 +-
.../types/EncodedValueTypeInformation.java | 4 -
.../flink/translation/types/FlinkCoder.java | 11 +-
.../flink/translation/types/KvKeySelector.java | 1 -
.../flink/translation/types/package-info.java | 22 +
.../utils/SerializedPipelineOptions.java | 6 +-
.../flink/translation/utils/package-info.java | 22 +
.../wrappers/DataInputViewWrapper.java | 3 +-
.../wrappers/DataOutputViewWrapper.java | 5 +-
.../SerializableFnAggregatorWrapper.java | 9 +-
.../translation/wrappers/SourceInputFormat.java | 6 +-
.../translation/wrappers/SourceInputSplit.java | 1 -
.../translation/wrappers/package-info.java | 22 +
.../wrappers/streaming/DoFnOperator.java | 512 ++++++
.../streaming/FlinkAbstractParDoWrapper.java | 280 ----
.../FlinkGroupAlsoByWindowWrapper.java | 642 --------
.../streaming/FlinkGroupByKeyWrapper.java | 73 -
.../streaming/FlinkParDoBoundMultiWrapper.java | 79 -
.../streaming/FlinkParDoBoundWrapper.java | 104 --
.../wrappers/streaming/FlinkStateInternals.java | 1037 ++++++++++++
.../streaming/SingletonKeyedWorkItem.java | 58 +
.../streaming/SingletonKeyedWorkItemCoder.java | 130 ++
.../wrappers/streaming/WindowDoFnOperator.java | 502 ++++++
.../wrappers/streaming/WorkItemKeySelector.java | 56 +
.../streaming/io/BoundedSourceWrapper.java | 223 +++
.../io/FlinkStreamingCreateFunction.java | 56 -
.../streaming/io/UnboundedFlinkSink.java | 27 +-
.../streaming/io/UnboundedFlinkSource.java | 36 +-
.../streaming/io/UnboundedSocketSource.java | 63 +-
.../streaming/io/UnboundedSourceWrapper.java | 202 ++-
.../wrappers/streaming/io/package-info.java | 22 +
.../wrappers/streaming/package-info.java | 22 +
.../state/AbstractFlinkTimerInternals.java | 127 --
.../streaming/state/FlinkStateInternals.java | 733 ---------
.../streaming/state/StateCheckpointReader.java | 93 --
.../streaming/state/StateCheckpointUtils.java | 155 --
.../streaming/state/StateCheckpointWriter.java | 131 --
.../wrappers/streaming/state/StateType.java | 73 -
.../flink/EncodedValueComparatorTest.java | 1 -
.../runners/flink/FlinkRunnerRegistrarTest.java | 1 -
.../beam/runners/flink/FlinkTestPipeline.java | 6 +-
.../beam/runners/flink/PipelineOptionsTest.java | 106 +-
.../beam/runners/flink/ReadSourceITCase.java | 15 +-
.../flink/ReadSourceStreamingITCase.java | 8 +-
.../beam/runners/flink/WriteSinkITCase.java | 16 +-
.../flink/streaming/DoFnOperatorTest.java | 326 ++++
.../streaming/FlinkStateInternalsTest.java | 390 +++++
.../flink/streaming/GroupAlsoByWindowTest.java | 523 ------
.../flink/streaming/GroupByNullKeyTest.java | 17 +-
.../flink/streaming/StateSerializationTest.java | 338 ----
.../flink/streaming/TestCountingSource.java | 12 +-
.../streaming/TopWikipediaSessionsITCase.java | 13 +-
.../streaming/UnboundedSourceWrapperTest.java | 158 +-
.../runners/flink/streaming/package-info.java | 22 +
runners/google-cloud-dataflow-java/pom.xml | 30 +-
.../dataflow/BlockingDataflowRunner.java | 18 +-
.../runners/dataflow/DataflowJobException.java | 1 -
.../runners/dataflow/DataflowPipelineJob.java | 174 +-
.../dataflow/DataflowPipelineRegistrar.java | 5 +-
.../dataflow/DataflowPipelineTranslator.java | 196 ++-
.../beam/runners/dataflow/DataflowRunner.java | 352 ++---
.../runners/dataflow/DataflowRunnerHooks.java | 3 +-
.../dataflow/internal/AssignWindows.java | 6 +-
.../dataflow/internal/CustomSources.java | 18 +-
.../internal/DataflowAggregatorTransforms.java | 10 +-
.../internal/DataflowMetricUpdateExtractor.java | 10 +-
.../DataflowUnboundedReadFromBoundedSource.java | 61 +-
.../runners/dataflow/internal/IsmFormat.java | 48 +-
.../dataflow/internal/ReadTranslator.java | 10 +-
.../dataflow/options/CloudDebuggerOptions.java | 6 +-
.../options/DataflowPipelineDebugOptions.java | 18 +-
.../options/DataflowPipelineOptions.java | 61 +-
.../DataflowPipelineWorkerPoolOptions.java | 30 +-
.../options/DataflowProfilingOptions.java | 5 +-
.../options/DataflowWorkerLoggingOptions.java | 12 +-
.../dataflow/testing/TestDataflowRunner.java | 171 +-
.../dataflow/util/DataflowTransport.java | 6 +-
.../beam/runners/dataflow/util/DoFnInfo.java | 38 +-
.../beam/runners/dataflow/util/GcsStager.java | 6 +-
.../runners/dataflow/util/MonitoringUtil.java | 16 +-
.../beam/runners/dataflow/util/PackageUtil.java | 47 +-
.../runners/dataflow/util/RandomAccessData.java | 24 +-
.../beam/runners/dataflow/util/Stager.java | 3 +-
.../beam/runners/dataflow/util/TimeUtil.java | 8 +-
.../dataflow/BlockingDataflowRunnerTest.java | 1 -
.../dataflow/DataflowPipelineJobTest.java | 189 ++-
.../dataflow/DataflowPipelineRegistrarTest.java | 9 +-
.../DataflowPipelineTranslatorTest.java | 78 +-
.../runners/dataflow/DataflowRunnerTest.java | 126 +-
.../dataflow/RecordingPipelineVisitor.java | 5 +-
...aflowUnboundedReadFromBoundedSourceTest.java | 83 +
.../DataflowPipelineDebugOptionsTest.java | 1 -
.../options/DataflowPipelineOptionsTest.java | 43 +-
.../options/DataflowProfilingOptionsTest.java | 4 +-
.../DataflowWorkerLoggingOptionsTest.java | 8 +-
.../testing/TestDataflowRunnerTest.java | 322 +++-
.../transforms/DataflowGroupByKeyTest.java | 6 +-
.../dataflow/transforms/DataflowViewTest.java | 1 -
.../dataflow/util/MonitoringUtilTest.java | 21 +-
.../runners/dataflow/util/PackageUtilTest.java | 41 +-
.../dataflow/util/RandomAccessDataTest.java | 13 +-
.../runners/dataflow/util/TimeUtilTest.java | 1 -
runners/pom.xml | 3 +-
runners/spark/README.md | 3 +-
runners/spark/pom.xml | 168 +-
.../beam/runners/spark/EvaluationResult.java | 4 +-
.../runners/spark/SparkPipelineOptions.java | 82 +-
.../apache/beam/runners/spark/SparkRunner.java | 195 ++-
.../runners/spark/SparkRunnerRegistrar.java | 5 +-
.../beam/runners/spark/TestSparkRunner.java | 19 +-
.../spark/aggregators/AccumulatorSingleton.java | 53 +
.../spark/aggregators/NamedAggregators.java | 48 +-
.../aggregators/metrics/AggregatorMetric.java | 44 +
.../metrics/AggregatorMetricSource.java | 50 +
.../metrics/WithNamedAggregatorsSupport.java | 174 ++
.../spark/aggregators/metrics/package-info.java | 22 +
.../spark/aggregators/metrics/sink/CsvSink.java | 39 +
.../aggregators/metrics/sink/GraphiteSink.java | 39 +
.../aggregators/metrics/sink/package-info.java | 23 +
.../coders/BeamSparkRunnerRegistrator.java | 46 +
.../beam/runners/spark/coders/CoderHelpers.java | 12 +-
.../runners/spark/coders/NullWritableCoder.java | 6 +-
.../runners/spark/coders/WritableCoder.java | 16 +-
.../beam/runners/spark/examples/WordCount.java | 4 +-
.../apache/beam/runners/spark/io/ConsoleIO.java | 2 +-
.../beam/runners/spark/io/CreateStream.java | 7 +-
.../runners/spark/io/EmptyCheckpointMark.java | 52 +
.../apache/beam/runners/spark/io/KafkaIO.java | 133 --
.../beam/runners/spark/io/MicrobatchSource.java | 262 +++
.../beam/runners/spark/io/SourceDStream.java | 156 ++
.../apache/beam/runners/spark/io/SourceRDD.java | 273 ++++
.../runners/spark/io/SparkUnboundedSource.java | 167 ++
.../beam/runners/spark/io/hadoop/HadoopIO.java | 16 +-
.../spark/io/hadoop/ShardNameBuilder.java | 3 +-
.../spark/io/hadoop/ShardNameTemplateAware.java | 2 +-
.../io/hadoop/ShardNameTemplateHelper.java | 9 +-
.../io/hadoop/TemplatedAvroKeyOutputFormat.java | 5 +-
.../TemplatedSequenceFileOutputFormat.java | 3 +-
.../io/hadoop/TemplatedTextOutputFormat.java | 3 +-
.../spark/stateful/StateSpecFunctions.java | 167 ++
.../runners/spark/stateful/package-info.java | 22 +
.../runners/spark/translation/DoFnFunction.java | 66 +-
.../spark/translation/EvaluationContext.java | 61 +-
.../translation/GroupCombineFunctions.java | 303 ++++
.../spark/translation/MultiDoFnFunction.java | 92 +-
.../translation/SparkAbstractCombineFn.java | 134 ++
.../spark/translation/SparkContextFactory.java | 53 +-
.../spark/translation/SparkGlobalCombineFn.java | 260 +++
.../spark/translation/SparkKeyedCombineFn.java | 273 ++++
.../translation/SparkPipelineEvaluator.java | 57 -
.../translation/SparkPipelineTranslator.java | 7 +-
.../spark/translation/SparkProcessContext.java | 196 ++-
.../spark/translation/SparkRuntimeContext.java | 70 +-
.../spark/translation/TransformEvaluator.java | 3 +-
.../spark/translation/TransformTranslator.java | 623 ++------
.../spark/translation/TranslationUtils.java | 214 +++
.../SparkRunnerStreamingContextFactory.java | 113 ++
.../streaming/StreamingEvaluationContext.java | 93 +-
.../streaming/StreamingTransformTranslator.java | 583 ++++---
.../StreamingWindowPipelineDetector.java | 102 --
.../runners/spark/util/BroadcastHelper.java | 16 +-
.../beam/runners/spark/util/ByteArray.java | 1 -
.../spark/util/SparkSideInputReader.java | 95 ++
.../apache/beam/runners/spark/DeDupTest.java | 62 -
.../beam/runners/spark/EmptyInputTest.java | 78 -
.../runners/spark/ProvidedSparkContextTest.java | 138 ++
.../beam/runners/spark/SimpleWordCountTest.java | 96 --
.../runners/spark/SparkRunnerRegistrarTest.java | 9 +-
.../apache/beam/runners/spark/TfIdfTest.java | 262 ---
.../metrics/sink/ClearAggregatorsRule.java | 33 +
.../metrics/sink/InMemoryMetrics.java | 86 +
.../metrics/sink/InMemoryMetricsSinkRule.java | 31 +
.../metrics/sink/NamedAggregatorsTest.java | 92 ++
.../runners/spark/coders/WritableCoderTest.java | 1 -
.../beam/runners/spark/io/AvroPipelineTest.java | 27 +-
.../beam/runners/spark/io/NumShardsTest.java | 25 +-
.../io/hadoop/HadoopFileFormatPipelineTest.java | 10 +-
.../spark/io/hadoop/ShardNameBuilderTest.java | 1 -
.../spark/translation/CombineGloballyTest.java | 105 --
.../spark/translation/CombinePerKeyTest.java | 82 -
.../spark/translation/DoFnOutputTest.java | 69 -
.../translation/MultiOutputWordCountTest.java | 179 ---
.../spark/translation/SerializationTest.java | 204 ---
.../spark/translation/SideEffectsTest.java | 27 +-
.../translation/SparkPipelineOptionsTest.java | 43 -
.../translation/TransformTranslatorTest.java | 107 --
.../translation/WindowedWordCountTest.java | 123 --
.../streaming/EmptyStreamAssertionTest.java | 80 +
.../streaming/FlattenStreamingTest.java | 64 +-
.../streaming/KafkaStreamingTest.java | 188 ++-
.../ResumeFromCheckpointStreamingTest.java | 188 +++
.../streaming/SimpleStreamingWordCountTest.java | 71 +-
.../streaming/utils/EmbeddedKafkaCluster.java | 63 +-
.../utils/KafkaWriteOnBatchCompleted.java | 105 ++
.../streaming/utils/PAssertStreaming.java | 89 +-
.../utils/TestOptionsForStreaming.java | 55 +
.../spark/src/test/resources/metrics.properties | 68 +
sdks/java/build-tools/pom.xml | 2 +-
.../src/main/resources/beam/checkstyle.xml | 39 +-
.../src/main/resources/beam/findbugs-filter.xml | 22 +-
.../src/main/resources/beam/suppressions.xml | 11 +-
sdks/java/core/pom.xml | 49 +-
.../beam/sdk/AggregatorPipelineExtractor.java | 91 ++
.../beam/sdk/AggregatorRetrievalException.java | 33 +
.../org/apache/beam/sdk/AggregatorValues.java | 51 +
.../main/java/org/apache/beam/sdk/Pipeline.java | 41 +-
.../org/apache/beam/sdk/PipelineResult.java | 31 +-
.../beam/sdk/annotations/Experimental.java | 15 +-
.../org/apache/beam/sdk/coders/AvroCoder.java | 35 +-
.../apache/beam/sdk/coders/BigDecimalCoder.java | 1 -
.../beam/sdk/coders/BigEndianIntegerCoder.java | 1 -
.../beam/sdk/coders/BigEndianLongCoder.java | 1 -
.../apache/beam/sdk/coders/BigIntegerCoder.java | 1 -
.../apache/beam/sdk/coders/ByteArrayCoder.java | 11 +-
.../org/apache/beam/sdk/coders/ByteCoder.java | 1 -
.../apache/beam/sdk/coders/ByteStringCoder.java | 7 +-
.../sdk/coders/CannotProvideCoderException.java | 2 +-
.../java/org/apache/beam/sdk/coders/Coder.java | 39 +-
.../apache/beam/sdk/coders/CoderFactories.java | 3 +-
.../apache/beam/sdk/coders/CoderFactory.java | 4 +-
.../apache/beam/sdk/coders/CoderProvider.java | 2 +-
.../apache/beam/sdk/coders/CoderProviders.java | 6 +-
.../apache/beam/sdk/coders/CoderRegistry.java | 24 +-
.../apache/beam/sdk/coders/CollectionCoder.java | 4 +-
.../org/apache/beam/sdk/coders/CustomCoder.java | 16 +-
.../apache/beam/sdk/coders/DefaultCoder.java | 3 +-
.../apache/beam/sdk/coders/DelegateCoder.java | 5 +-
.../org/apache/beam/sdk/coders/DoubleCoder.java | 1 -
.../apache/beam/sdk/coders/DurationCoder.java | 8 +-
.../apache/beam/sdk/coders/InstantCoder.java | 12 +-
.../apache/beam/sdk/coders/IterableCoder.java | 9 +-
.../beam/sdk/coders/IterableLikeCoder.java | 31 +-
.../org/apache/beam/sdk/coders/JAXBCoder.java | 14 +-
.../org/apache/beam/sdk/coders/KvCoder.java | 13 +-
.../org/apache/beam/sdk/coders/ListCoder.java | 4 +-
.../org/apache/beam/sdk/coders/MapCoder.java | 9 +-
.../apache/beam/sdk/coders/NullableCoder.java | 29 +-
.../beam/sdk/coders/SerializableCoder.java | 6 +-
.../org/apache/beam/sdk/coders/SetCoder.java | 4 +-
.../apache/beam/sdk/coders/StandardCoder.java | 25 +-
.../beam/sdk/coders/StringDelegateCoder.java | 5 +-
.../apache/beam/sdk/coders/StringUtf8Coder.java | 11 +-
.../beam/sdk/coders/TableRowJsonCoder.java | 4 +-
.../beam/sdk/coders/TextualIntegerCoder.java | 1 -
.../org/apache/beam/sdk/coders/VarIntCoder.java | 4 +-
.../apache/beam/sdk/coders/VarLongCoder.java | 4 +-
.../org/apache/beam/sdk/coders/VoidCoder.java | 1 -
.../apache/beam/sdk/coders/package-info.java | 2 +-
.../beam/sdk/coders/protobuf/ProtoCoder.java | 32 +-
.../beam/sdk/coders/protobuf/ProtobufUtil.java | 4 +-
.../java/org/apache/beam/sdk/io/AvroIO.java | 299 +++-
.../java/org/apache/beam/sdk/io/AvroSource.java | 143 +-
.../apache/beam/sdk/io/BlockBasedSource.java | 6 +-
.../sdk/io/BoundedReadFromUnboundedSource.java | 40 +-
.../org/apache/beam/sdk/io/BoundedSource.java | 30 +-
.../apache/beam/sdk/io/CompressedSource.java | 114 +-
.../org/apache/beam/sdk/io/CountingInput.java | 4 +-
.../org/apache/beam/sdk/io/CountingSource.java | 11 +-
.../org/apache/beam/sdk/io/FileBasedSink.java | 172 +-
.../org/apache/beam/sdk/io/FileBasedSource.java | 17 +-
.../apache/beam/sdk/io/OffsetBasedSource.java | 16 +-
.../java/org/apache/beam/sdk/io/PubsubIO.java | 50 +-
.../apache/beam/sdk/io/PubsubUnboundedSink.java | 23 +-
.../beam/sdk/io/PubsubUnboundedSource.java | 49 +-
.../main/java/org/apache/beam/sdk/io/Read.java | 18 +-
.../sdk/io/SerializableAvroCodecFactory.java | 112 ++
.../main/java/org/apache/beam/sdk/io/Sink.java | 8 +-
.../java/org/apache/beam/sdk/io/Source.java | 8 +-
.../java/org/apache/beam/sdk/io/TextIO.java | 331 ++--
.../org/apache/beam/sdk/io/UnboundedSource.java | 11 +-
.../main/java/org/apache/beam/sdk/io/Write.java | 50 +-
.../java/org/apache/beam/sdk/io/XmlSink.java | 14 +-
.../java/org/apache/beam/sdk/io/XmlSource.java | 28 +-
.../org/apache/beam/sdk/io/range/ByteKey.java | 1 -
.../apache/beam/sdk/io/range/ByteKeyRange.java | 12 +-
.../beam/sdk/io/range/ByteKeyRangeTracker.java | 6 +-
.../beam/sdk/io/range/OffsetRangeTracker.java | 4 +-
.../apache/beam/sdk/io/range/RangeTracker.java | 1 +
.../org/apache/beam/sdk/metrics/Counter.java | 40 +
.../apache/beam/sdk/metrics/CounterCell.java | 76 +
.../org/apache/beam/sdk/metrics/DirtyState.java | 98 ++
.../apache/beam/sdk/metrics/Distribution.java | 30 +
.../beam/sdk/metrics/DistributionCell.java | 58 +
.../beam/sdk/metrics/DistributionData.java | 59 +
.../beam/sdk/metrics/DistributionResult.java | 42 +
.../org/apache/beam/sdk/metrics/Metric.java | 24 +
.../org/apache/beam/sdk/metrics/MetricCell.java | 47 +
.../org/apache/beam/sdk/metrics/MetricKey.java | 40 +
.../org/apache/beam/sdk/metrics/MetricName.java | 46 +
.../beam/sdk/metrics/MetricNameFilter.java | 60 +
.../beam/sdk/metrics/MetricQueryResults.java | 33 +
.../apache/beam/sdk/metrics/MetricResult.java | 45 +
.../apache/beam/sdk/metrics/MetricResults.java | 34 +
.../apache/beam/sdk/metrics/MetricUpdates.java | 72 +
.../org/apache/beam/sdk/metrics/Metrics.java | 110 ++
.../beam/sdk/metrics/MetricsContainer.java | 150 ++
.../beam/sdk/metrics/MetricsEnvironment.java | 85 +
.../apache/beam/sdk/metrics/MetricsFilter.java | 86 +
.../org/apache/beam/sdk/metrics/MetricsMap.java | 86 +
.../apache/beam/sdk/metrics/package-info.java | 28 +
.../options/CloudResourceManagerOptions.java | 40 +
.../org/apache/beam/sdk/options/GcpOptions.java | 26 +-
.../org/apache/beam/sdk/options/GcsOptions.java | 19 +-
.../beam/sdk/options/GoogleApiDebugOptions.java | 3 +-
.../beam/sdk/options/PipelineOptionSpec.java | 4 +-
.../beam/sdk/options/PipelineOptions.java | 111 +-
.../sdk/options/PipelineOptionsFactory.java | 295 ++--
.../sdk/options/PipelineOptionsReflector.java | 4 +-
.../sdk/options/PipelineOptionsRegistrar.java | 1 -
.../sdk/options/PipelineOptionsValidator.java | 6 +-
.../sdk/options/ProxyInvocationHandler.java | 302 ++--
.../apache/beam/sdk/options/ValueProvider.java | 239 +++
.../beam/sdk/options/ValueProviderUtils.java | 60 +
.../java/org/apache/beam/sdk/package-info.java | 2 +-
.../runners/AggregatorPipelineExtractor.java | 93 --
.../runners/AggregatorRetrievalException.java | 33 -
.../beam/sdk/runners/AggregatorValues.java | 52 -
.../sdk/runners/PipelineRunnerRegistrar.java | 3 +-
.../beam/sdk/runners/TransformHierarchy.java | 9 +-
.../beam/sdk/runners/TransformTreeNode.java | 14 +-
.../apache/beam/sdk/runners/package-info.java | 17 +-
.../beam/sdk/testing/BigqueryMatcher.java | 239 +++
.../beam/sdk/testing/CoderProperties.java | 23 +-
.../beam/sdk/testing/FileChecksumMatcher.java | 133 ++
.../beam/sdk/testing/MatcherDeserializer.java | 7 +-
.../beam/sdk/testing/MatcherSerializer.java | 7 +-
.../org/apache/beam/sdk/testing/PAssert.java | 68 +-
.../apache/beam/sdk/testing/PaneExtractors.java | 23 +-
.../beam/sdk/testing/SerializableMatcher.java | 3 +-
.../beam/sdk/testing/SerializableMatchers.java | 22 +-
.../beam/sdk/testing/SourceTestUtils.java | 150 +-
.../apache/beam/sdk/testing/StaticWindows.java | 10 +-
.../apache/beam/sdk/testing/StreamingIT.java | 35 +
.../apache/beam/sdk/testing/TestPipeline.java | 32 +-
.../beam/sdk/testing/TestPipelineOptions.java | 1 -
.../org/apache/beam/sdk/testing/TestStream.java | 368 +++++
.../beam/sdk/testing/WindowFnTestUtils.java | 19 +-
.../apache/beam/sdk/testing/WindowSupplier.java | 10 +-
.../apache/beam/sdk/testing/package-info.java | 4 +-
.../apache/beam/sdk/transforms/Aggregator.java | 25 +-
.../beam/sdk/transforms/AppliedPTransform.java | 5 +-
.../sdk/transforms/ApproximateQuantiles.java | 34 +-
.../beam/sdk/transforms/ApproximateUnique.java | 23 +-
.../org/apache/beam/sdk/transforms/Combine.java | 201 ++-
.../beam/sdk/transforms/CombineFnBase.java | 27 +-
.../apache/beam/sdk/transforms/CombineFns.java | 82 +-
.../beam/sdk/transforms/CombineWithContext.java | 7 +-
.../org/apache/beam/sdk/transforms/Count.java | 21 +-
.../org/apache/beam/sdk/transforms/Create.java | 44 +-
.../sdk/transforms/DelegatingAggregator.java | 125 ++
.../org/apache/beam/sdk/transforms/DoFn.java | 538 ++++++-
.../beam/sdk/transforms/DoFnAdapters.java | 440 ++++++
.../beam/sdk/transforms/DoFnReflector.java | 1070 -------------
.../apache/beam/sdk/transforms/DoFnTester.java | 391 +++--
.../org/apache/beam/sdk/transforms/Filter.java | 4 +-
.../beam/sdk/transforms/FlatMapElements.java | 13 +-
.../org/apache/beam/sdk/transforms/Flatten.java | 12 +-
.../apache/beam/sdk/transforms/GroupByKey.java | 7 +-
.../transforms/IntraBundleParallelization.java | 364 -----
.../org/apache/beam/sdk/transforms/Keys.java | 8 +-
.../org/apache/beam/sdk/transforms/KvSwap.java | 9 +-
.../org/apache/beam/sdk/transforms/Latest.java | 203 +++
.../apache/beam/sdk/transforms/MapElements.java | 24 +-
.../org/apache/beam/sdk/transforms/Max.java | 32 +-
.../org/apache/beam/sdk/transforms/Mean.java | 12 +-
.../org/apache/beam/sdk/transforms/Min.java | 33 +-
.../org/apache/beam/sdk/transforms/OldDoFn.java | 356 ++++-
.../apache/beam/sdk/transforms/PTransform.java | 19 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 616 +++++---
.../apache/beam/sdk/transforms/Partition.java | 7 +-
.../beam/sdk/transforms/RemoveDuplicates.java | 13 +-
.../org/apache/beam/sdk/transforms/Sample.java | 7 +-
.../sdk/transforms/SerializableFunction.java | 2 +-
.../beam/sdk/transforms/SimpleFunction.java | 20 +-
.../org/apache/beam/sdk/transforms/Sum.java | 27 +-
.../org/apache/beam/sdk/transforms/Top.java | 20 +-
.../org/apache/beam/sdk/transforms/Values.java | 8 +-
.../org/apache/beam/sdk/transforms/View.java | 5 +-
.../org/apache/beam/sdk/transforms/ViewFn.java | 44 +
.../apache/beam/sdk/transforms/WithKeys.java | 11 +-
.../beam/sdk/transforms/WithTimestamps.java | 1 -
.../sdk/transforms/display/DisplayData.java | 549 ++++---
.../beam/sdk/transforms/join/CoGbkResult.java | 30 +-
.../sdk/transforms/join/CoGbkResultSchema.java | 10 +-
.../beam/sdk/transforms/join/CoGroupByKey.java | 5 +-
.../transforms/join/KeyedPCollectionTuple.java | 7 +-
.../beam/sdk/transforms/join/RawUnionValue.java | 25 +
.../beam/sdk/transforms/join/UnionCoder.java | 14 +-
.../sdk/transforms/reflect/DoFnInvoker.java | 73 +
.../sdk/transforms/reflect/DoFnInvokers.java | 781 +++++++++
.../sdk/transforms/reflect/DoFnSignature.java | 520 ++++++
.../sdk/transforms/reflect/DoFnSignatures.java | 1023 ++++++++++++
.../sdk/transforms/reflect/package-info.java | 23 +
.../splittabledofn/RestrictionTracker.java | 42 +
.../transforms/splittabledofn/package-info.java | 22 +
.../beam/sdk/transforms/windowing/AfterAll.java | 60 +-
.../windowing/AfterDelayFromFirstElement.java | 122 +-
.../sdk/transforms/windowing/AfterEach.java | 70 +-
.../sdk/transforms/windowing/AfterFirst.java | 61 +-
.../sdk/transforms/windowing/AfterPane.java | 66 +-
.../windowing/AfterProcessingTime.java | 14 +-
.../AfterSynchronizedProcessingTime.java | 23 +-
.../transforms/windowing/AfterWatermark.java | 186 +--
.../sdk/transforms/windowing/BoundedWindow.java | 3 +-
.../transforms/windowing/CalendarWindows.java | 1 -
.../transforms/windowing/DefaultTrigger.java | 41 +-
.../sdk/transforms/windowing/FixedWindows.java | 4 +-
.../sdk/transforms/windowing/GlobalWindow.java | 6 +-
.../sdk/transforms/windowing/GlobalWindows.java | 6 +-
.../transforms/windowing/IntervalWindow.java | 10 +-
.../transforms/windowing/InvalidWindows.java | 4 +-
.../beam/sdk/transforms/windowing/Never.java | 32 +-
.../transforms/windowing/OrFinallyTrigger.java | 60 +-
.../sdk/transforms/windowing/OutputTimeFn.java | 7 +-
.../sdk/transforms/windowing/OutputTimeFns.java | 7 +-
.../beam/sdk/transforms/windowing/PaneInfo.java | 28 +-
.../windowing/PartitioningWindowFn.java | 3 +-
.../sdk/transforms/windowing/Repeatedly.java | 40 +-
.../beam/sdk/transforms/windowing/Sessions.java | 8 +-
.../transforms/windowing/SlidingWindows.java | 13 +-
.../beam/sdk/transforms/windowing/Trigger.java | 429 +----
.../beam/sdk/transforms/windowing/Window.java | 34 +-
.../beam/sdk/transforms/windowing/WindowFn.java | 8 +-
.../apache/beam/sdk/util/ActiveWindowSet.java | 6 +-
.../org/apache/beam/sdk/util/ApiSurface.java | 6 +-
.../apache/beam/sdk/util/AppliedCombineFn.java | 6 +-
...AttemptAndTimeBoundedExponentialBackOff.java | 7 +-
.../util/AttemptBoundedExponentialBackOff.java | 1 +
.../org/apache/beam/sdk/util/AvroUtils.java | 7 +-
.../beam/sdk/util/BaseExecutionContext.java | 17 +-
.../org/apache/beam/sdk/util/BitSetCoder.java | 11 +-
.../apache/beam/sdk/util/BucketingFunction.java | 3 +-
.../BufferedElementCountingOutputStream.java | 4 +-
.../apache/beam/sdk/util/CloudKnownType.java | 1 -
.../org/apache/beam/sdk/util/CloudObject.java | 2 -
.../org/apache/beam/sdk/util/CoderUtils.java | 17 +-
.../org/apache/beam/sdk/util/CombineFnUtil.java | 7 +-
.../apache/beam/sdk/util/CounterAggregator.java | 128 --
.../apache/beam/sdk/util/CredentialFactory.java | 3 +-
.../org/apache/beam/sdk/util/Credentials.java | 9 +-
.../beam/sdk/util/DirectSideInputReader.java | 9 +-
.../apache/beam/sdk/util/ExecutableTrigger.java | 47 +-
.../apache/beam/sdk/util/ExecutionContext.java | 5 +-
.../sdk/util/ExposedByteArrayInputStream.java | 3 +-
.../sdk/util/ExposedByteArrayOutputStream.java | 4 +-
.../beam/sdk/util/FileIOChannelFactory.java | 6 +-
.../apache/beam/sdk/util/FinishedTriggers.java | 44 -
.../beam/sdk/util/FinishedTriggersBitSet.java | 67 -
.../beam/sdk/util/FinishedTriggersSet.java | 73 -
.../org/apache/beam/sdk/util/FluentBackoff.java | 229 +++
.../apache/beam/sdk/util/GatherAllPanes.java | 10 +-
.../beam/sdk/util/GcpCredentialFactory.java | 6 +-
.../apache/beam/sdk/util/GcpProjectUtil.java | 106 ++
.../beam/sdk/util/GcsIOChannelFactory.java | 5 +-
.../apache/beam/sdk/util/GcsPathValidator.java | 5 +-
.../java/org/apache/beam/sdk/util/GcsUtil.java | 367 +++--
.../apache/beam/sdk/util/IOChannelFactory.java | 2 +-
.../apache/beam/sdk/util/IOChannelUtils.java | 5 +-
.../apache/beam/sdk/util/IdentityWindowFn.java | 6 +-
.../apache/beam/sdk/util/InstanceBuilder.java | 5 +-
.../util/IntervalBoundedExponentialBackOff.java | 1 +
.../beam/sdk/util/KeyedWorkItemCoder.java | 17 +-
.../apache/beam/sdk/util/KeyedWorkItems.java | 4 +-
.../beam/sdk/util/MapAggregatorValues.java | 50 -
.../beam/sdk/util/MergingActiveWindowSet.java | 33 +-
.../apache/beam/sdk/util/MovingFunction.java | 3 +-
.../apache/beam/sdk/util/MutationDetectors.java | 5 +-
.../sdk/util/NonMergingActiveWindowSet.java | 6 +-
.../beam/sdk/util/NoopCredentialFactory.java | 4 +-
.../beam/sdk/util/NullSideInputReader.java | 6 +-
.../beam/sdk/util/PCollectionViewWindow.java | 3 +-
.../apache/beam/sdk/util/PCollectionViews.java | 248 ++-
.../java/org/apache/beam/sdk/util/PTuple.java | 3 +-
.../org/apache/beam/sdk/util/PathValidator.java | 6 +-
.../beam/sdk/util/PerKeyCombineFnRunner.java | 31 +-
.../beam/sdk/util/PerKeyCombineFnRunners.java | 10 +-
.../org/apache/beam/sdk/util/PropertyNames.java | 1 +
.../org/apache/beam/sdk/util/PubsubClient.java | 10 +-
.../apache/beam/sdk/util/PubsubGrpcClient.java | 42 +-
.../apache/beam/sdk/util/PubsubJsonClient.java | 5 +-
.../apache/beam/sdk/util/PubsubTestClient.java | 7 +-
.../sdk/util/ReifyTimestampAndWindowsDoFn.java | 20 +-
.../org/apache/beam/sdk/util/ReleaseInfo.java | 12 +-
.../org/apache/beam/sdk/util/Reshuffle.java | 7 +-
.../apache/beam/sdk/util/ReshuffleTrigger.java | 20 +-
.../sdk/util/RetryHttpRequestInitializer.java | 7 +-
.../apache/beam/sdk/util/SerializableUtils.java | 15 +-
.../org/apache/beam/sdk/util/Serializer.java | 2 -
.../apache/beam/sdk/util/SideInputReader.java | 3 +-
.../org/apache/beam/sdk/util/StringUtils.java | 6 +-
.../java/org/apache/beam/sdk/util/Structs.java | 2 -
.../beam/sdk/util/SystemDoFnInternal.java | 8 +-
.../apache/beam/sdk/util/TestCredential.java | 1 -
.../java/org/apache/beam/sdk/util/Timer.java | 56 +
.../apache/beam/sdk/util/TimerInternals.java | 45 +-
.../org/apache/beam/sdk/util/TimerSpec.java | 30 +
.../org/apache/beam/sdk/util/TimerSpecs.java | 41 +
.../java/org/apache/beam/sdk/util/Timers.java | 14 +-
.../org/apache/beam/sdk/util/Transport.java | 25 +-
.../beam/sdk/util/TriggerContextFactory.java | 511 ------
.../beam/sdk/util/UnownedInputStream.java | 1 -
.../beam/sdk/util/UnownedOutputStream.java | 1 -
.../sdk/util/UploadIdResponseInterceptor.java | 4 +-
.../apache/beam/sdk/util/ValueWithRecordId.java | 19 +-
.../java/org/apache/beam/sdk/util/Values.java | 1 -
.../org/apache/beam/sdk/util/WindowedValue.java | 32 +-
.../beam/sdk/util/WindowingInternals.java | 9 +-
.../apache/beam/sdk/util/WindowingStrategy.java | 11 +-
.../java/org/apache/beam/sdk/util/ZipFiles.java | 4 +-
.../apache/beam/sdk/util/common/Counter.java | 1287 ---------------
.../beam/sdk/util/common/CounterName.java | 153 --
.../beam/sdk/util/common/CounterProvider.java | 27 -
.../apache/beam/sdk/util/common/CounterSet.java | 179 ---
.../util/common/ElementByteSizeObserver.java | 24 +-
.../beam/sdk/util/common/ReflectHelpers.java | 25 -
.../apache/beam/sdk/util/common/Reiterable.java | 2 +-
.../apache/beam/sdk/util/common/Reiterator.java | 2 +-
.../org/apache/beam/sdk/util/gcsfs/GcsPath.java | 2 -
.../CopyOnAccessInMemoryStateInternals.java | 18 +-
.../sdk/util/state/InMemoryStateInternals.java | 13 +-
.../sdk/util/state/InMemoryTimerInternals.java | 235 +++
.../sdk/util/state/MergingStateAccessor.java | 3 +-
.../beam/sdk/util/state/ReadableState.java | 10 +-
.../apache/beam/sdk/util/state/StateBinder.java | 67 +
.../beam/sdk/util/state/StateContext.java | 6 +-
.../beam/sdk/util/state/StateContexts.java | 3 +-
.../sdk/util/state/StateInternalsFactory.java | 3 +-
.../beam/sdk/util/state/StateMerging.java | 9 +-
.../beam/sdk/util/state/StateNamespaces.java | 10 +-
.../apache/beam/sdk/util/state/StateSpec.java | 39 +
.../apache/beam/sdk/util/state/StateSpecs.java | 452 ++++++
.../apache/beam/sdk/util/state/StateTable.java | 4 +-
.../apache/beam/sdk/util/state/StateTag.java | 87 +-
.../apache/beam/sdk/util/state/StateTags.java | 396 +----
.../util/state/TestInMemoryStateInternals.java | 61 +
.../beam/sdk/util/state/TimerCallback.java | 35 +
.../beam/sdk/util/state/WatermarkHoldState.java | 1 -
.../java/org/apache/beam/sdk/values/KV.java | 10 +-
.../java/org/apache/beam/sdk/values/PBegin.java | 5 +-
.../apache/beam/sdk/values/PCollectionList.java | 12 +-
.../beam/sdk/values/PCollectionTuple.java | 12 +-
.../apache/beam/sdk/values/PCollectionView.java | 33 +-
.../java/org/apache/beam/sdk/values/PDone.java | 5 +-
.../java/org/apache/beam/sdk/values/PInput.java | 11 +-
.../org/apache/beam/sdk/values/POutput.java | 11 +-
.../java/org/apache/beam/sdk/values/PValue.java | 4 +-
.../org/apache/beam/sdk/values/PValueBase.java | 5 +-
.../beam/sdk/values/TimestampedValue.java | 26 +-
.../org/apache/beam/sdk/values/TupleTag.java | 13 +-
.../apache/beam/sdk/values/TupleTagList.java | 4 +-
.../apache/beam/sdk/values/TypeDescriptor.java | 2 -
.../apache/beam/sdk/values/TypeDescriptors.java | 41 +-
.../apache/beam/sdk/values/package-info.java | 4 +-
.../dataflow/util/GcsPathValidatorTest.java | 104 --
.../sdk/AggregatorPipelineExtractorTest.java | 226 +++
.../org/apache/beam/sdk/DataflowMatchers.java | 66 -
.../java/org/apache/beam/sdk/PipelineTest.java | 18 +-
.../java/org/apache/beam/sdk/TestUtils.java | 10 +-
.../org/apache/beam/sdk/WindowMatchers.java | 9 +-
.../org/apache/beam/sdk/WindowMatchersTest.java | 4 +-
.../apache/beam/sdk/coders/AvroCoderTest.java | 64 +-
.../beam/sdk/coders/BigDecimalCoderTest.java | 6 +-
.../sdk/coders/BigEndianIntegerCoderTest.java | 6 +-
.../beam/sdk/coders/BigEndianLongCoderTest.java | 6 +-
.../beam/sdk/coders/BigIntegerCoderTest.java | 6 +-
.../beam/sdk/coders/ByteArrayCoderTest.java | 10 +-
.../apache/beam/sdk/coders/ByteCoderTest.java | 6 +-
.../beam/sdk/coders/ByteStringCoderTest.java | 11 +-
.../beam/sdk/coders/CoderFactoriesTest.java | 5 +-
.../beam/sdk/coders/CoderProvidersTest.java | 4 +-
.../beam/sdk/coders/CoderRegistryTest.java | 32 +-
.../org/apache/beam/sdk/coders/CoderTest.java | 4 +-
.../beam/sdk/coders/CollectionCoderTest.java | 14 +-
.../apache/beam/sdk/coders/CustomCoderTest.java | 12 +-
.../beam/sdk/coders/DefaultCoderTest.java | 7 +-
.../beam/sdk/coders/DelegateCoderTest.java | 11 +-
.../apache/beam/sdk/coders/DoubleCoderTest.java | 6 +-
.../beam/sdk/coders/DurationCoderTest.java | 9 +-
.../beam/sdk/coders/InstantCoderTest.java | 15 +-
.../beam/sdk/coders/IterableCoderTest.java | 10 +-
.../apache/beam/sdk/coders/JAXBCoderTest.java | 29 +-
.../org/apache/beam/sdk/coders/KvCoderTest.java | 108 +-
.../apache/beam/sdk/coders/ListCoderTest.java | 10 +-
.../apache/beam/sdk/coders/MapCoderTest.java | 17 +-
.../beam/sdk/coders/NullableCoderTest.java | 73 +-
.../beam/sdk/coders/PrintBase64Encodings.java | 4 +-
.../beam/sdk/coders/SerializableCoderTest.java | 14 +-
.../apache/beam/sdk/coders/SetCoderTest.java | 12 +-
.../beam/sdk/coders/StandardCoderTest.java | 35 +-
.../sdk/coders/StringDelegateCoderTest.java | 8 +-
.../beam/sdk/coders/StringUtf8CoderTest.java | 6 +-
.../beam/sdk/coders/TableRowJsonCoderTest.java | 9 +-
.../sdk/coders/TextualIntegerCoderTest.java | 6 +-
.../apache/beam/sdk/coders/VarIntCoderTest.java | 6 +-
.../beam/sdk/coders/VarLongCoderTest.java | 6 +-
.../sdk/coders/protobuf/ProtoCoderTest.java | 14 +-
.../sdk/coders/protobuf/ProtobufUtilTest.java | 10 +-
.../beam/sdk/io/AvroIOGeneratedClassTest.java | 26 +-
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 168 +-
.../org/apache/beam/sdk/io/AvroSourceTest.java | 89 +-
.../io/BoundedReadFromUnboundedSourceTest.java | 25 +-
.../beam/sdk/io/CompressedSourceTest.java | 47 +-
.../apache/beam/sdk/io/CountingInputTest.java | 2 -
.../apache/beam/sdk/io/CountingSourceTest.java | 6 +-
.../sdk/io/DrunkWritableByteChannelFactory.java | 80 +
.../apache/beam/sdk/io/FileBasedSinkTest.java | 130 +-
.../apache/beam/sdk/io/FileBasedSourceTest.java | 32 +-
.../beam/sdk/io/OffsetBasedSourceTest.java | 41 +-
.../org/apache/beam/sdk/io/PubsubIOTest.java | 7 +-
.../beam/sdk/io/PubsubUnboundedSinkTest.java | 13 +-
.../beam/sdk/io/PubsubUnboundedSourceTest.java | 19 +-
.../java/org/apache/beam/sdk/io/ReadTest.java | 74 +-
.../io/SerializableAvroCodecFactoryTest.java | 100 ++
.../java/org/apache/beam/sdk/io/TextIOTest.java | 702 ++++++---
.../java/org/apache/beam/sdk/io/WriteTest.java | 86 +-
.../org/apache/beam/sdk/io/XmlSinkTest.java | 27 +-
.../org/apache/beam/sdk/io/XmlSourceTest.java | 29 +-
.../range/ByteKeyRangeEstimateFractionTest.java | 1 -
.../range/ByteKeyRangeInterpolateKeyTest.java | 1 -
.../beam/sdk/io/range/ByteKeyRangeTest.java | 6 +-
.../apache/beam/sdk/io/range/ByteKeyTest.java | 3 +-
.../beam/sdk/metrics/CounterCellTest.java | 55 +
.../apache/beam/sdk/metrics/DirtyStateTest.java | 56 +
.../beam/sdk/metrics/DistributionCellTest.java | 53 +
.../apache/beam/sdk/metrics/MetricMatchers.java | 99 ++
.../beam/sdk/metrics/MetricsContainerTest.java | 129 ++
.../sdk/metrics/MetricsEnvironmentTest.java | 63 +
.../apache/beam/sdk/metrics/MetricsMapTest.java | 103 ++
.../apache/beam/sdk/metrics/MetricsTest.java | 98 ++
.../apache/beam/sdk/options/GcpOptionsTest.java | 16 +-
.../sdk/options/GoogleApiDebugOptionsTest.java | 8 +-
.../sdk/options/PipelineOptionsFactoryTest.java | 211 ++-
.../options/PipelineOptionsReflectorTest.java | 6 +-
.../beam/sdk/options/PipelineOptionsTest.java | 55 +-
.../options/PipelineOptionsValidatorTest.java | 19 +-
.../sdk/options/ProxyInvocationHandlerTest.java | 171 +-
.../beam/sdk/options/ValueProviderTest.java | 213 +++
.../sdk/options/ValueProviderUtilsTest.java | 78 +
.../AggregatorPipelineExtractorTest.java | 229 ---
.../beam/sdk/runners/PipelineRunnerTest.java | 1 -
.../beam/sdk/runners/TransformTreeTest.java | 8 +-
.../runners/dataflow/TestCountingSource.java | 13 +-
.../dataflow/TestCountingSourceTest.java | 4 +-
.../beam/sdk/testing/BigqueryMatcherTest.java | 176 +++
.../beam/sdk/testing/CoderPropertiesTest.java | 11 +-
.../beam/sdk/testing/CrashingRunnerTest.java | 1 -
.../apache/beam/sdk/testing/ExpectedLogs.java | 12 +-
.../beam/sdk/testing/ExpectedLogsTest.java | 13 +-
.../sdk/testing/FastNanoClockAndSleeper.java | 1 -
.../testing/FastNanoClockAndSleeperTest.java | 3 +-
.../sdk/testing/FileChecksumMatcherTest.java | 104 ++
.../apache/beam/sdk/testing/PAssertTest.java | 17 +-
.../sdk/testing/PCollectionViewTesting.java | 47 +-
.../beam/sdk/testing/PaneExtractorsTest.java | 4 +-
.../sdk/testing/RestoreSystemProperties.java | 5 +-
.../sdk/testing/SerializableMatchersTest.java | 12 +-
.../beam/sdk/testing/SourceTestUtilsTest.java | 63 +
.../beam/sdk/testing/StaticWindowsTest.java | 4 +-
.../beam/sdk/testing/SystemNanoTimeSleeper.java | 3 +-
.../sdk/testing/SystemNanoTimeSleeperTest.java | 1 -
.../beam/sdk/testing/TestPipelineTest.java | 16 +-
.../apache/beam/sdk/testing/TestStreamTest.java | 355 +++++
.../beam/sdk/testing/WindowSupplierTest.java | 16 +-
.../transforms/ApproximateQuantilesTest.java | 13 +-
.../sdk/transforms/ApproximateUniqueTest.java | 18 +-
.../beam/sdk/transforms/CombineFnsTest.java | 25 +-
.../apache/beam/sdk/transforms/CombineTest.java | 149 +-
.../apache/beam/sdk/transforms/CountTest.java | 7 +-
.../apache/beam/sdk/transforms/CreateTest.java | 26 +-
.../DoFnDelegatingAggregatorTest.java | 6 +-
.../beam/sdk/transforms/DoFnReflectorTest.java | 686 --------
.../apache/beam/sdk/transforms/DoFnTest.java | 34 +-
.../beam/sdk/transforms/DoFnTesterTest.java | 469 +++---
.../apache/beam/sdk/transforms/FilterTest.java | 5 +-
.../sdk/transforms/FlatMapElementsTest.java | 35 +-
.../apache/beam/sdk/transforms/FlattenTest.java | 41 +-
.../beam/sdk/transforms/GroupByKeyTest.java | 30 +-
.../IntraBundleParallelizationTest.java | 283 ----
.../apache/beam/sdk/transforms/KeysTest.java | 4 +-
.../apache/beam/sdk/transforms/KvSwapTest.java | 4 +-
.../beam/sdk/transforms/LatestFnTest.java | 233 +++
.../apache/beam/sdk/transforms/LatestTest.java | 146 ++
.../beam/sdk/transforms/MapElementsTest.java | 23 +-
.../org/apache/beam/sdk/transforms/MaxTest.java | 5 +-
.../apache/beam/sdk/transforms/MeanTest.java | 10 +-
.../org/apache/beam/sdk/transforms/MinTest.java | 5 +-
.../apache/beam/sdk/transforms/NoOpOldDoFn.java | 3 +-
.../apache/beam/sdk/transforms/OldDoFnTest.java | 10 +-
.../beam/sdk/transforms/PTransformTest.java | 1 -
.../beam/sdk/transforms/ParDoLifecycleTest.java | 447 ++++++
.../apache/beam/sdk/transforms/ParDoTest.java | 141 +-
.../beam/sdk/transforms/PartitionTest.java | 9 +-
.../sdk/transforms/RemoveDuplicatesTest.java | 10 +-
.../apache/beam/sdk/transforms/SampleTest.java | 21 +-
.../beam/sdk/transforms/SimpleStatsFnsTest.java | 7 +-
.../org/apache/beam/sdk/transforms/SumTest.java | 5 +-
.../org/apache/beam/sdk/transforms/TopTest.java | 13 +-
.../apache/beam/sdk/transforms/ValuesTest.java | 4 +-
.../apache/beam/sdk/transforms/ViewTest.java | 219 ++-
.../beam/sdk/transforms/WithKeysTest.java | 6 +-
.../beam/sdk/transforms/WithTimestampsTest.java | 4 +-
.../display/DisplayDataEvaluator.java | 21 +-
.../display/DisplayDataEvaluatorTest.java | 7 +-
.../transforms/display/DisplayDataMatchers.java | 148 +-
.../display/DisplayDataMatchersTest.java | 69 +-
.../sdk/transforms/display/DisplayDataTest.java | 393 +++--
.../dofnreflector/DoFnReflectorTestHelper.java | 116 --
.../transforms/join/CoGbkResultCoderTest.java | 4 +-
.../sdk/transforms/join/CoGbkResultTest.java | 6 +-
.../sdk/transforms/join/CoGroupByKeyTest.java | 15 +-
.../sdk/transforms/join/UnionCoderTest.java | 4 +-
.../transforms/reflect/DoFnInvokersTest.java | 598 +++++++
.../DoFnSignaturesProcessElementTest.java | 213 +++
.../DoFnSignaturesSplittableDoFnTest.java | 543 +++++++
.../transforms/reflect/DoFnSignaturesTest.java | 883 +++++++++++
.../reflect/DoFnSignaturesTestUtils.java | 67 +
.../testhelper/DoFnInvokersTestHelper.java | 124 ++
.../sdk/transforms/windowing/AfterAllTest.java | 99 --
.../sdk/transforms/windowing/AfterEachTest.java | 65 -
.../transforms/windowing/AfterFirstTest.java | 121 --
.../sdk/transforms/windowing/AfterPaneTest.java | 78 -
.../windowing/AfterProcessingTimeTest.java | 95 --
.../AfterSynchronizedProcessingTimeTest.java | 76 -
.../windowing/AfterWatermarkTest.java | 309 ----
.../windowing/CalendarWindowsTest.java | 13 +-
.../windowing/DefaultTriggerTest.java | 131 --
.../transforms/windowing/FixedWindowsTest.java | 11 +-
.../windowing/IntervalWindowTest.java | 7 +-
.../sdk/transforms/windowing/NeverTest.java | 35 +-
.../windowing/OrFinallyTriggerTest.java | 137 --
.../sdk/transforms/windowing/PaneInfoTest.java | 1 -
.../transforms/windowing/RepeatedlyTest.java | 162 +-
.../sdk/transforms/windowing/SessionsTest.java | 16 +-
.../windowing/SlidingWindowsTest.java | 11 +-
.../sdk/transforms/windowing/StubTrigger.java | 21 +-
.../sdk/transforms/windowing/TriggerTest.java | 33 +-
.../sdk/transforms/windowing/WindowTest.java | 9 +-
.../sdk/transforms/windowing/WindowingTest.java | 10 +-
.../apache/beam/sdk/util/ApiSurfaceTest.java | 38 +-
...mptAndTimeBoundedExponentialBackOffTest.java | 5 +-
.../AttemptBoundedExponentialBackOffTest.java | 2 +-
.../org/apache/beam/sdk/util/AvroUtilsTest.java | 20 +-
.../beam/sdk/util/BucketingFunctionTest.java | 1 -
...BufferedElementCountingOutputStreamTest.java | 19 +-
.../apache/beam/sdk/util/CoderUtilsTest.java | 7 +-
.../apache/beam/sdk/util/CombineFnUtilTest.java | 13 +-
.../beam/sdk/util/CounterAggregatorTest.java | 256 ---
.../beam/sdk/util/ExecutableTriggerTest.java | 24 +-
.../util/ExposedByteArrayInputStreamTest.java | 5 +-
.../util/ExposedByteArrayOutputStreamTest.java | 5 +-
.../beam/sdk/util/FileIOChannelFactoryTest.java | 16 +-
.../sdk/util/FinishedTriggersBitSetTest.java | 55 -
.../sdk/util/FinishedTriggersProperties.java | 110 --
.../beam/sdk/util/FinishedTriggersSetTest.java | 61 -
.../apache/beam/sdk/util/FluentBackoffTest.java | 226 +++
.../beam/sdk/util/GatherAllPanesTest.java | 7 +-
.../beam/sdk/util/GcpProjectUtilTest.java | 76 +
.../beam/sdk/util/GcsIOChannelFactoryTest.java | 1 -
.../beam/sdk/util/GcsPathValidatorTest.java | 100 ++
.../org/apache/beam/sdk/util/GcsUtilTest.java | 214 ++-
.../beam/sdk/util/IOChannelUtilsTest.java | 6 +-
.../sdk/util/IdentitySideInputWindowFn.java | 5 +-
.../beam/sdk/util/InstanceBuilderTest.java | 1 -
.../beam/sdk/util/KeyedWorkItemCoderTest.java | 4 +-
.../sdk/util/MergingActiveWindowSetTest.java | 13 +-
.../beam/sdk/util/MovingFunctionTest.java | 1 -
.../beam/sdk/util/MutationDetectorsTest.java | 17 +-
.../org/apache/beam/sdk/util/PTupleTest.java | 1 -
.../apache/beam/sdk/util/PubsubClientTest.java | 10 +-
.../beam/sdk/util/PubsubGrpcClientTest.java | 126 +-
.../beam/sdk/util/PubsubJsonClientTest.java | 34 +-
.../beam/sdk/util/PubsubTestClientTest.java | 18 +-
.../org/apache/beam/sdk/util/ReshuffleTest.java | 7 +-
.../beam/sdk/util/ReshuffleTriggerTest.java | 24 -
.../util/RetryHttpRequestInitializerTest.java | 12 +-
.../beam/sdk/util/SerializableUtilsTest.java | 19 +-
.../apache/beam/sdk/util/StreamUtilsTest.java | 9 +-
.../apache/beam/sdk/util/StringUtilsTest.java | 1 -
.../org/apache/beam/sdk/util/StructsTest.java | 11 +-
.../beam/sdk/util/TimerInternalsTest.java | 48 +-
.../org/apache/beam/sdk/util/TriggerTester.java | 597 -------
.../beam/sdk/util/UnownedInputStreamTest.java | 3 +-
.../beam/sdk/util/UnownedOutputStreamTest.java | 3 +-
.../util/UploadIdResponseInterceptorTest.java | 7 +-
.../beam/sdk/util/UserCodeExceptionTest.java | 3 +-
.../org/apache/beam/sdk/util/VarIntTest.java | 9 +-
.../apache/beam/sdk/util/WindowedValueTest.java | 9 +-
.../org/apache/beam/sdk/util/ZipFilesTest.java | 14 +-
.../beam/sdk/util/common/CounterSetTest.java | 227 ---
.../beam/sdk/util/common/CounterTest.java | 736 ---------
.../sdk/util/common/ReflectHelpersTest.java | 6 +-
.../apache/beam/sdk/util/gcsfs/GcsPathTest.java | 11 +-
.../CopyOnAccessInMemoryStateInternalsTest.java | 1 -
.../util/state/InMemoryStateInternalsTest.java | 4 +-
.../util/state/InMemoryTimerInternalsTest.java | 116 ++
.../sdk/util/state/StateNamespacesTest.java | 1 -
.../beam/sdk/util/state/StateTagTest.java | 1 -
.../java/org/apache/beam/sdk/values/KVTest.java | 4 +-
.../beam/sdk/values/PCollectionListTest.java | 3 +-
.../beam/sdk/values/PCollectionTupleTest.java | 8 +-
.../org/apache/beam/sdk/values/PDoneTest.java | 4 +-
.../beam/sdk/values/TimestampedValueTest.java | 83 +
.../beam/sdk/values/TypeDescriptorTest.java | 10 +-
.../beam/sdk/values/TypeDescriptorsTest.java | 6 +-
.../apache/beam/sdk/values/TypedPValueTest.java | 1 -
sdks/java/extensions/join-library/pom.xml | 2 +-
.../extensions/joinlibrary/InnerJoinTest.java | 6 +-
.../joinlibrary/OuterLeftJoinTest.java | 6 +-
.../joinlibrary/OuterRightJoinTest.java | 6 +-
sdks/java/extensions/pom.xml | 2 +-
sdks/java/io/google-cloud-platform/pom.xml | 20 +-
.../sdk/io/gcp/bigquery/BigQueryAvroUtils.java | 86 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 448 +++---
.../sdk/io/gcp/bigquery/BigQueryServices.java | 20 +-
.../io/gcp/bigquery/BigQueryServicesImpl.java | 271 ++--
.../io/gcp/bigquery/BigQueryTableInserter.java | 14 +-
.../gcp/bigquery/BigQueryTableRowIterator.java | 145 +-
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 116 +-
.../sdk/io/gcp/bigtable/BigtableService.java | 15 +-
.../io/gcp/bigtable/BigtableServiceImpl.java | 20 +-
.../beam/sdk/io/gcp/datastore/DatastoreIO.java | 14 +-
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 1028 ++++++++++++
.../beam/sdk/io/gcp/datastore/V1Beta3.java | 969 ------------
.../beam/sdk/io/gcp/datastore/package-info.java | 2 +-
.../io/gcp/bigquery/BigQueryAvroUtilsTest.java | 160 +-
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 869 ++++++++--
.../gcp/bigquery/BigQueryServicesImplTest.java | 193 ++-
.../gcp/bigquery/BigQueryTableInserterTest.java | 20 +-
.../bigquery/BigQueryTableRowIteratorTest.java | 179 ++-
.../sdk/io/gcp/bigquery/BigQueryUtilTest.java | 19 +-
.../sdk/io/gcp/bigtable/BigtableIOTest.java | 160 +-
.../sdk/io/gcp/bigtable/BigtableReadIT.java | 6 +-
.../sdk/io/gcp/bigtable/BigtableWriteIT.java | 29 +-
.../sdk/io/gcp/datastore/DatastoreV1Test.java | 837 ++++++++++
.../sdk/io/gcp/datastore/SplitQueryFnIT.java | 97 ++
.../sdk/io/gcp/datastore/V1Beta3ReadIT.java | 114 --
.../beam/sdk/io/gcp/datastore/V1Beta3Test.java | 587 -------
.../io/gcp/datastore/V1Beta3TestOptions.java | 44 -
.../sdk/io/gcp/datastore/V1Beta3TestUtil.java | 382 -----
.../sdk/io/gcp/datastore/V1Beta3WriteIT.java | 85 -
.../beam/sdk/io/gcp/datastore/V1ReadIT.java | 111 ++
.../sdk/io/gcp/datastore/V1TestOptions.java | 43 +
.../beam/sdk/io/gcp/datastore/V1TestUtil.java | 382 +++++
.../beam/sdk/io/gcp/datastore/V1WriteIT.java | 83 +
sdks/java/io/hdfs/pom.xml | 7 +-
.../beam/sdk/io/hdfs/AvroHDFSFileSource.java | 25 +-
.../beam/sdk/io/hdfs/AvroWrapperCoder.java | 18 +-
.../apache/beam/sdk/io/hdfs/HDFSFileSink.java | 17 +-
.../apache/beam/sdk/io/hdfs/HDFSFileSource.java | 40 +-
.../apache/beam/sdk/io/hdfs/WritableCoder.java | 14 +-
.../SimpleAuthAvroHDFSFileSource.java | 14 +-
.../hdfs/simpleauth/SimpleAuthHDFSFileSink.java | 5 +-
.../simpleauth/SimpleAuthHDFSFileSource.java | 19 +-
.../beam/sdk/io/hdfs/AvroWrapperCoderTest.java | 5 +-
.../beam/sdk/io/hdfs/HDFSFileSourceTest.java | 12 +-
.../beam/sdk/io/hdfs/WritableCoderTest.java | 1 -
sdks/java/io/jdbc/pom.xml | 138 ++
.../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 443 ++++++
.../apache/beam/sdk/io/jdbc/package-info.java | 22 +
.../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 271 ++++
sdks/java/io/jms/pom.xml | 2 +-
.../beam/sdk/io/jms/JmsCheckpointMark.java | 9 +-
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 58 +-
.../org/apache/beam/sdk/io/jms/JmsRecord.java | 1 -
.../org/apache/beam/sdk/io/jms/JmsIOTest.java | 25 +-
sdks/java/io/kafka/pom.xml | 2 +-
.../beam/sdk/io/kafka/KafkaCheckpointMark.java | 16 +-
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 189 ++-
.../apache/beam/sdk/io/kafka/KafkaRecord.java | 3 +-
.../beam/sdk/io/kafka/KafkaRecordCoder.java | 14 +-
.../apache/beam/sdk/io/kafka/KafkaIOTest.java | 199 ++-
sdks/java/io/kinesis/pom.xml | 178 +++
.../sdk/io/kinesis/CheckpointGenerator.java | 30 +
.../beam/sdk/io/kinesis/CustomOptional.java | 86 +
.../io/kinesis/DynamicCheckpointGenerator.java | 56 +
.../sdk/io/kinesis/GetKinesisRecordsResult.java | 54 +
.../sdk/io/kinesis/KinesisClientProvider.java | 31 +
.../apache/beam/sdk/io/kinesis/KinesisIO.java | 192 +++
.../beam/sdk/io/kinesis/KinesisReader.java | 145 ++
.../sdk/io/kinesis/KinesisReaderCheckpoint.java | 96 ++
.../beam/sdk/io/kinesis/KinesisRecord.java | 121 ++
.../beam/sdk/io/kinesis/KinesisRecordCoder.java | 74 +
.../beam/sdk/io/kinesis/KinesisSource.java | 112 ++
.../beam/sdk/io/kinesis/RecordFilter.java | 41 +
.../apache/beam/sdk/io/kinesis/RoundRobin.java | 53 +
.../beam/sdk/io/kinesis/ShardCheckpoint.java | 175 ++
.../sdk/io/kinesis/ShardRecordsIterator.java | 98 ++
.../sdk/io/kinesis/SimplifiedKinesisClient.java | 157 ++
.../beam/sdk/io/kinesis/StartingPoint.java | 85 +
.../io/kinesis/StaticCheckpointGenerator.java | 42 +
.../io/kinesis/TransientKinesisException.java | 29 +
.../beam/sdk/io/kinesis/package-info.java | 22 +
.../beam/sdk/io/kinesis/AmazonKinesisMock.java | 375 +++++
.../beam/sdk/io/kinesis/CustomOptionalTest.java | 31 +
.../kinesis/DynamicCheckpointGeneratorTest.java | 57 +
.../sdk/io/kinesis/KinesisMockReadTest.java | 91 ++
.../io/kinesis/KinesisReaderCheckpointTest.java | 67 +
.../beam/sdk/io/kinesis/KinesisReaderIT.java | 119 ++
.../beam/sdk/io/kinesis/KinesisReaderTest.java | 120 ++
.../sdk/io/kinesis/KinesisRecordCoderTest.java | 45 +
.../beam/sdk/io/kinesis/KinesisTestOptions.java | 47 +
.../beam/sdk/io/kinesis/KinesisUploader.java | 83 +
.../beam/sdk/io/kinesis/RecordFilterTest.java | 66 +
.../beam/sdk/io/kinesis/RoundRobinTest.java | 57 +
.../sdk/io/kinesis/ShardCheckpointTest.java | 149 ++
.../io/kinesis/ShardRecordsIteratorTest.java | 151 ++
.../io/kinesis/SimplifiedKinesisClientTest.java | 224 +++
sdks/java/io/mongodb/pom.xml | 139 ++
.../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 449 ++++++
.../apache/beam/sdk/io/mongodb/MongoDbIO.java | 503 ++++++
.../beam/sdk/io/mongodb/package-info.java | 22 +
.../sdk/io/mongodb/MongoDBGridFSIOTest.java | 276 ++++
.../beam/sdk/io/mongodb/MongoDbIOTest.java | 208 +++
sdks/java/io/pom.xml | 5 +-
sdks/java/java8tests/pom.xml | 2 +-
.../PipelineOptionsFactoryJava8Test.java | 8 +-
.../beam/sdk/transforms/CombineJava8Test.java | 7 +-
.../beam/sdk/transforms/FilterJava8Test.java | 4 +-
.../transforms/FlatMapElementsJava8Test.java | 9 +-
.../sdk/transforms/MapElementsJava8Test.java | 4 +-
.../beam/sdk/transforms/PartitionJava8Test.java | 4 +-
.../transforms/RemoveDuplicatesJava8Test.java | 11 +-
.../beam/sdk/transforms/WithKeysJava8Test.java | 1 -
.../sdk/transforms/WithTimestampsJava8Test.java | 10 +-
sdks/java/maven-archetypes/examples/pom.xml | 27 +-
.../main/resources/archetype-resources/pom.xml | 26 +-
.../src/main/java/DebuggingWordCount.java | 41 +-
.../src/main/java/MinimalWordCount.java | 50 +-
.../src/main/java/WindowedWordCount.java | 156 +-
.../src/main/java/WordCount.java | 79 +-
.../java/common/DataflowExampleOptions.java | 32 -
.../main/java/common/DataflowExampleUtils.java | 394 -----
.../common/ExampleBigQueryTableOptions.java | 14 +-
.../src/main/java/common/ExampleOptions.java | 32 +
...xamplePubsubTopicAndSubscriptionOptions.java | 45 +
.../java/common/ExamplePubsubTopicOptions.java | 17 +-
.../src/main/java/common/ExampleUtils.java | 353 +++++
.../main/java/common/PubsubFileInjector.java | 155 --
.../src/test/java/DebuggingWordCountTest.java | 6 +-
.../src/test/java/WordCountTest.java | 14 +-
sdks/java/maven-archetypes/pom.xml | 2 +-
sdks/java/maven-archetypes/starter/pom.xml | 10 +-
.../main/resources/archetype-resources/pom.xml | 8 +-
.../src/main/java/StarterPipeline.java | 1 -
.../resources/projects/basic/reference/pom.xml | 8 +-
.../src/main/java/it/pkg/StarterPipeline.java | 1 -
sdks/java/microbenchmarks/pom.xml | 2 +-
.../coders/AvroCoderBenchmark.java | 6 +-
.../coders/ByteArrayCoderBenchmark.java | 6 +-
.../coders/CoderBenchmarking.java | 3 +-
.../coders/StringUtf8CoderBenchmark.java | 6 +-
.../transforms/DoFnInvokersBenchmark.java | 231 +++
.../transforms/DoFnReflectorBenchmark.java | 243 ---
sdks/java/pom.xml | 2 +-
sdks/pom.xml | 2 +-
testing/travis/README.md | 23 -
testing/travis/test_wordcount.sh | 125 --
1287 files changed, 69862 insertions(+), 41390 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/02b5b4c9/.travis.yml
----------------------------------------------------------------------
diff --cc .travis.yml
index d9d5e1e,6a8b604..5ca7f3a
--- a/.travis.yml
+++ b/.travis.yml
@@@ -28,45 -28,47 +28,50 @@@ notifications
on_success: change
on_failure: always
+addons:
+ apt:
+ packages:
+ - python2.7
+ env:
+ global:
+ - MAVEN_OVERRIDE="--settings=.travis/settings.xml -Djavadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true"
+ - MAVEN_CONTAINER_OVERRIDE="-DbeamSurefireArgline='-Xmx512m'"
matrix:
include:
- # TODO(altay): Re-enable Java tests before merging python-sdk branch to master.
- ## On OSX, run with default JDK only.
- #- os: osx
- # env: MAVEN_OVERRIDE=""
- ## On Linux, run with specific JDKs only.
- #- os: linux
- # env: CUSTOM_JDK="oraclejdk8" MAVEN_OVERRIDE="-DbeamSurefireArgline='-Xmx512m'"
- #- os: linux
- # env: CUSTOM_JDK="oraclejdk7" MAVEN_OVERRIDE="-DbeamSurefireArgline='-Xmx512m'"
- #- os: linux
- # env: CUSTOM_JDK="openjdk7" MAVEN_OVERRIDE="-DbeamSurefireArgline='-Xmx512m'"
- # Python SDK environments.
- # On OSX, run with default JDK only.
- os: osx
-
- # On Linux, run with specific JDKs only.
- - os: linux
- env: CUSTOM_JDK="oraclejdk8" MAVEN_OVERRIDE="$MAVEN_OVERRIDE $MAVEN_CONTAINER_OVERRIDE"
- - os: linux
- env: CUSTOM_JDK="oraclejdk7" MAVEN_OVERRIDE="$MAVEN_OVERRIDE $MAVEN_CONTAINER_OVERRIDE"
- - os: linux
- env: CUSTOM_JDK="oraclejdk7" MAVEN_OVERRIDE="$MAVEN_OVERRIDE $MAVEN_CONTAINER_OVERRIDE -Peclipse-jdt"
+ env: TEST_PYTHON="1"
- os: linux
- env: CUSTOM_JDK="openjdk7" MAVEN_OVERRIDE="$MAVEN_OVERRIDE $MAVEN_CONTAINER_OVERRIDE"
+ env: TEST_PYTHON="1"
before_install:
- - echo "MAVEN_OPTS='-Xmx2048m -XX:MaxPermSize=512m'" > ~/.mavenrc
+ - echo 'MAVEN_OPTS="$MAVEN_OPTS -Xmx1024m -XX:MaxPermSize=512m -XX:+BytecodeVerificationLocal"' >> ~/.mavenrc
+ - echo $'MAVEN_OPTS="$MAVEN_OPTS -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd\'T\'HH:mm:ss.SSS"' >> ~/.mavenrc
+ - cat ~/.mavenrc
- if [ "$TRAVIS_OS_NAME" == "osx" ]; then export JAVA_HOME=$(/usr/libexec/java_home); fi
- if [ "$TRAVIS_OS_NAME" == "linux" ]; then jdk_switcher use "$CUSTOM_JDK"; fi
- - export BEAM_SUREFIRE_ARGLINE="-Xmx512m"
+ # Python SDK environment settings.
+ - export TOX_ENV=py27
+ - if [ "$TRAVIS_OS_NAME" == "osx" ]; then export TOX_HOME=$HOME/Library/Python/2.7/bin; fi
+ - if [ "$TRAVIS_OS_NAME" == "linux" ]; then export TOX_HOME=$HOME/.local/bin; fi
install:
+ - if [ ! "$TEST_PYTHON" ]; then travis_retry mvn -B install clean -U -DskipTests=true; fi
+ - if [ "$TEST_PYTHON" ] && pip list | grep tox; then TOX_FILE=`which tox` ; export TOX_HOME=`dirname $TOX_FILE`; fi
+ - if [ "$TEST_PYTHON" ] && ! pip list | grep tox; then travis_retry pip install tox --user `whoami`; fi
+ # Removing this here protects from inadvertent caching
+ - rm -rf "$HOME/.m2/repository/org/apache/beam"
script:
- - travis_retry mvn --batch-mode --update-snapshots --no-snapshot-updates $MAVEN_OVERRIDE install && travis_retry bash -ex .travis/test_wordcount.sh
+ - if [ "$TEST_PYTHON" ]; then travis_retry $TOX_HOME/tox -e $TOX_ENV -c sdks/python/tox.ini; fi
- - if [ ! "$TEST_PYTHON" ]; then travis_retry mvn -B $MAVEN_OVERRIDE install -U; fi
- - if [ ! "$TEST_PYTHON" ]; then travis_retry testing/travis/test_wordcount.sh; fi
++ - if [ ! "$TEST_PYTHON" ]; then travis_retry mvn --batch-mode --update-snapshots --no-snapshot-updates $MAVEN_OVERRIDE install && travis_retry bash -ex .travis/test_wordcount.sh; fi
+
+ cache:
+ directories:
+ - $HOME/.m2/repository
+
+ before_cache:
+ # Removing here increases cache hits (makes the above
+ # rm in `install` redundant unless our config has a bug,
+ # but it will be idempotent)
+ - rm -rf "$HOME/.m2/repository/org/apache/beam"
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/02b5b4c9/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/02b5b4c9/sdks/pom.xml
----------------------------------------------------------------------