You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2017/07/24 04:35:32 UTC
[49/50] [abbrv] beam git commit: Merge branch master up to commit
'5e3c5c6574bc70320683d6c16fc3b11791a77418' into jstorm-runner
Merge branch master up to commit '5e3c5c6574bc70320683d6c16fc3b11791a77418' into jstorm-runner
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/1fe64def
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/1fe64def
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/1fe64def
Branch: refs/heads/jstorm-runner
Commit: 1fe64def2f6a69b7f9f230c53237bc96dc28cb63
Parents: 58d4b97 5e3c5c6
Author: basti.lj <ba...@alibaba-inc.com>
Authored: Mon Jul 24 11:33:40 2017 +0800
Committer: basti.lj <ba...@alibaba-inc.com>
Committed: Mon Jul 24 11:33:40 2017 +0800
----------------------------------------------------------------------
.github/PULL_REQUEST_TEMPLATE.md | 3 +-
.gitignore | 3 +
.../jenkins/common_job_properties.groovy | 30 +-
.../job_beam_PerformanceTests_Dataflow.groovy | 3 +
.../job_beam_PerformanceTests_JDBC.groovy | 3 +
...job_beam_PostCommit_Java_MavenInstall.groovy | 8 +-
..._PostCommit_Java_MavenInstall_Windows.groovy | 45 +
...tCommit_Java_ValidatesRunner_Dataflow.groovy | 2 +-
...PostCommit_Java_ValidatesRunner_Flink.groovy | 2 +-
.../job_beam_PreCommit_Java_MavenInstall.groovy | 5 +-
.../job_beam_Release_NightlySnapshot.groovy | 2 +-
.test-infra/jenkins/test_wordcount.sh | 125 +
.test-infra/travis/README.md | 23 -
.test-infra/travis/settings.xml | 33 -
.test-infra/travis/test_wordcount.sh | 125 -
.travis.yml | 93 -
README.md | 7 +-
examples/java/README.md | 64 +-
examples/java/pom.xml | 90 +-
.../beam/examples/DebuggingWordCount.java | 22 +-
.../apache/beam/examples/MinimalWordCount.java | 7 +-
.../apache/beam/examples/WindowedWordCount.java | 11 +-
.../org/apache/beam/examples/WordCount.java | 20 +-
.../common/ExampleBigQueryTableOptions.java | 2 +-
...xamplePubsubTopicAndSubscriptionOptions.java | 2 +-
.../common/ExamplePubsubTopicOptions.java | 2 +-
.../beam/examples/common/ExampleUtils.java | 74 +-
.../examples/common/WriteOneFilePerWindow.java | 69 +-
.../beam/examples/complete/AutoComplete.java | 2 +-
.../org/apache/beam/examples/complete/README.md | 2 +-
.../examples/complete/StreamingWordExtract.java | 4 +-
.../apache/beam/examples/complete/TfIdf.java | 8 +-
.../examples/complete/TopWikipediaSessions.java | 4 +-
.../examples/complete/TrafficMaxLaneFlow.java | 2 +-
.../beam/examples/complete/TrafficRoutes.java | 2 +-
.../cookbook/CombinePerKeyExamples.java | 13 +-
.../beam/examples/cookbook/DistinctExample.java | 4 +-
.../beam/examples/cookbook/JoinExamples.java | 2 +-
.../org/apache/beam/examples/cookbook/README.md | 2 +-
.../beam/examples/cookbook/TriggerExample.java | 6 +-
.../beam/examples/WindowedWordCountIT.java | 41 +-
.../org/apache/beam/examples/WordCountIT.java | 14 +-
.../beam/examples/complete/TfIdfTest.java | 2 +-
.../examples/cookbook/BigQueryTornadoesIT.java | 4 +-
examples/java8/pom.xml | 135 +-
.../beam/examples/MinimalWordCountJava8.java | 7 +-
.../beam/examples/complete/game/GameStats.java | 33 +-
.../examples/complete/game/HourlyTeamScore.java | 60 +-
.../examples/complete/game/LeaderBoard.java | 68 +-
.../beam/examples/complete/game/README.md | 131 -
.../beam/examples/complete/game/UserScore.java | 82 +-
.../complete/game/utils/WriteToBigQuery.java | 32 +-
.../complete/game/utils/WriteToText.java | 184 ++
.../game/utils/WriteWindowedToBigQuery.java | 9 +-
.../examples/MinimalWordCountJava8Test.java | 6 +-
examples/pom.xml | 2 +-
pom.xml | 169 +-
runners/apex/README.md | 76 -
runners/apex/pom.xml | 17 +-
.../beam/runners/apex/ApexPipelineOptions.java | 5 +
.../apache/beam/runners/apex/ApexRunner.java | 51 +-
.../beam/runners/apex/ApexRunnerRegistrar.java | 3 +-
.../beam/runners/apex/ApexRunnerResult.java | 11 -
.../beam/runners/apex/ApexYarnLauncher.java | 2 -
.../beam/runners/apex/TestApexRunner.java | 10 +-
.../translation/ApexPipelineTranslator.java | 23 +
.../apex/translation/ParDoTranslator.java | 68 +-
.../translation/ReadUnboundedTranslator.java | 1 -
.../apex/translation/TransformTranslator.java | 2 -
.../apex/translation/TranslationContext.java | 97 +-
.../operators/ApexGroupByKeyOperator.java | 197 +-
.../operators/ApexParDoOperator.java | 281 +-
.../operators/ApexProcessFnOperator.java | 8 +-
.../ApexReadUnboundedInputOperator.java | 2 -
.../operators/ApexTimerInternals.java | 281 ++
.../translation/utils/ApexStateInternals.java | 124 +-
.../apex/translation/utils/ApexStreamTuple.java | 18 +-
.../utils/CoderAdapterStreamCodec.java | 2 -
.../apex/translation/utils/NoOpStepContext.java | 2 +-
.../utils/SerializablePipelineOptions.java | 19 +-
.../translation/utils/StateInternalsProxy.java | 17 +-
.../utils/ValueAndCoderKryoSerializable.java | 2 -
.../apex/translation/utils/ValuesSource.java | 4 +-
.../beam/runners/apex/ApexRunnerTest.java | 49 +-
.../beam/runners/apex/ApexYarnLauncherTest.java | 2 -
.../apex/examples/UnboundedTextSource.java | 4 +-
.../runners/apex/examples/WordCountTest.java | 15 +-
.../translation/ApexGroupByKeyOperatorTest.java | 4 +-
.../FlattenPCollectionTranslatorTest.java | 13 +-
.../translation/GroupByKeyTranslatorTest.java | 12 +-
.../apex/translation/ParDoTranslatorTest.java | 2 +-
.../translation/ReadUnboundTranslatorTest.java | 6 +-
.../operators/ApexTimerInternalsTest.java | 145 +
.../utils/ApexStateInternalsTest.java | 51 +-
.../translation/utils/CollectionSource.java | 4 +-
.../translation/utils/PipelineOptionsTest.java | 98 +-
.../test/resources/beam-runners-apex.properties | 2 +-
runners/core-construction-java/pom.xml | 14 +-
.../core/construction/CoderTranslator.java | 44 +
.../core/construction/CoderTranslators.java | 107 +
.../beam/runners/core/construction/Coders.java | 109 +-
.../DeduplicatedFlattenFactory.java | 2 +-
.../EmptyFlattenAsCreateFactory.java | 2 +-
.../core/construction/ForwardingPTransform.java | 11 +-
.../runners/core/construction/PCollections.java | 2 +-
.../core/construction/PTransformMatchers.java | 8 +-
.../construction/PTransformReplacements.java | 2 +-
.../runners/core/construction/PTransforms.java | 110 +
.../core/construction/PrimitiveCreate.java | 4 +-
.../core/construction/SdkComponents.java | 100 +-
.../runners/core/construction/Triggers.java | 4 +-
.../UnboundedReadFromBoundedSource.java | 34 +-
.../core/construction/UnconsumedReads.java | 72 +
.../UnsupportedOverrideFactory.java | 2 +-
.../core/construction/WindowingStrategies.java | 56 +-
.../runners/core/metrics/MetricFiltering.java | 102 +
.../beam/runners/core/metrics/MetricKey.java | 43 +
.../beam/runners/core/metrics/package-info.java | 22 +
.../runners/core/construction/CodersTest.java | 55 +-
.../EmptyFlattenAsCreateFactoryTest.java | 12 +-
.../construction/ForwardingPTransformTest.java | 7 +-
.../core/construction/PCollectionsTest.java | 23 +-
.../construction/PTransformMatchersTest.java | 54 +-
.../PTransformReplacementsTest.java | 6 +-
.../core/construction/PTransformsTest.java | 188 ++
.../construction/ReplacementOutputsTest.java | 2 +-
.../core/construction/SdkComponentsTest.java | 192 +-
.../SingleInputOutputOverrideFactoryTest.java | 2 +-
.../UnboundedReadFromBoundedSourceTest.java | 14 +-
.../core/construction/UnconsumedReadsTest.java | 105 +
.../construction/WindowingStrategiesTest.java | 10 +-
.../core/metrics/MetricFilteringTest.java | 148 +
runners/core-java/pom.xml | 17 +-
.../beam/runners/core/AggregatorFactory.java | 38 -
.../beam/runners/core/BaseExecutionContext.java | 15 +-
.../apache/beam/runners/core/DoFnRunner.java | 2 +-
.../apache/beam/runners/core/DoFnRunners.java | 78 +-
.../core/ElementAndRestrictionCoder.java | 29 +-
.../beam/runners/core/ExecutionContext.java | 12 +-
.../runners/core/GlobalCombineFnRunner.java | 77 +
.../runners/core/GlobalCombineFnRunners.java | 220 ++
.../GroupAlsoByWindowViaOutputBufferDoFn.java | 125 -
.../core/GroupAlsoByWindowViaWindowSetDoFn.java | 108 -
.../GroupAlsoByWindowViaWindowSetNewDoFn.java | 18 +-
.../core/GroupAlsoByWindowsAggregators.java | 4 +-
.../core/GroupByKeyViaGroupByKeyOnly.java | 2 +-
.../runners/core/InMemoryStateInternals.java | 134 +-
.../runners/core/InMemoryTimerInternals.java | 2 +-
.../beam/runners/core/KeyedWorkItemCoder.java | 40 +-
.../core/LateDataDroppingDoFnRunner.java | 26 +-
.../apache/beam/runners/core/LateDataUtils.java | 41 +-
.../runners/core/MergingActiveWindowSet.java | 6 +-
.../beam/runners/core/MergingStateAccessor.java | 4 +-
.../apache/beam/runners/core/NonEmptyPanes.java | 10 +-
.../beam/runners/core/NullSideInputReader.java | 61 +
.../org/apache/beam/runners/core/OldDoFn.java | 466 ---
...eBoundedSplittableProcessElementInvoker.java | 26 +-
.../beam/runners/core/PaneInfoTracker.java | 6 +-
.../runners/core/PerKeyCombineFnRunner.java | 79 -
.../runners/core/PerKeyCombineFnRunners.java | 161 -
.../beam/runners/core/ProcessFnRunner.java | 126 +
.../core/PushbackSideInputDoFnRunner.java | 108 +-
.../core/ReadyCheckingSideInputReader.java | 34 +
.../org/apache/beam/runners/core/ReduceFn.java | 6 +-
.../runners/core/ReduceFnContextFactory.java | 50 +-
.../beam/runners/core/ReduceFnRunner.java | 77 +-
.../beam/runners/core/SideInputHandler.java | 29 +-
.../beam/runners/core/SideInputReader.java | 47 +
.../beam/runners/core/SimpleDoFnRunner.java | 252 +-
.../beam/runners/core/SimpleOldDoFnRunner.java | 522 ----
.../core/SimplePushbackSideInputDoFnRunner.java | 114 +
.../beam/runners/core/SplittableParDo.java | 182 +-
.../apache/beam/runners/core/StateAccessor.java | 4 +-
.../beam/runners/core/StateInternals.java | 12 +-
.../runners/core/StateInternalsFactory.java | 2 +-
.../apache/beam/runners/core/StateMerging.java | 58 +-
.../beam/runners/core/StateNamespaces.java | 13 +-
.../apache/beam/runners/core/StateTable.java | 14 +-
.../org/apache/beam/runners/core/StateTag.java | 67 +-
.../org/apache/beam/runners/core/StateTags.java | 147 +-
.../beam/runners/core/StatefulDoFnRunner.java | 32 +-
.../beam/runners/core/SystemReduceFn.java | 29 +-
.../core/TestInMemoryStateInternals.java | 12 +-
.../beam/runners/core/TimerInternals.java | 44 +-
.../core/UnsupportedSideInputReader.java | 1 -
.../apache/beam/runners/core/WatermarkHold.java | 60 +-
.../beam/runners/core/WindowingInternals.java | 2 +-
.../core/WindowingInternalsAdapters.java | 75 -
.../beam/runners/core/metrics/CounterCell.java | 86 +
.../beam/runners/core/metrics/DirtyState.java | 99 +
.../runners/core/metrics/DistributionCell.java | 80 +
.../runners/core/metrics/DistributionData.java | 62 +
.../beam/runners/core/metrics/GaugeCell.java | 78 +
.../beam/runners/core/metrics/GaugeData.java | 83 +
.../beam/runners/core/metrics/MetricCell.java | 42 +
.../runners/core/metrics/MetricUpdates.java | 79 +
.../core/metrics/MetricsContainerImpl.java | 188 ++
.../core/metrics/MetricsContainerStepMap.java | 495 ++++
.../beam/runners/core/metrics/MetricsMap.java | 88 +
.../beam/runners/core/metrics/package-info.java | 22 +
.../AfterDelayFromFirstElementStateMachine.java | 8 +-
.../core/triggers/AfterPaneStateMachine.java | 4 +-
.../AfterProcessingTimeStateMachine.java | 2 +-
...rSynchronizedProcessingTimeStateMachine.java | 2 +-
.../triggers/AfterWatermarkStateMachine.java | 2 +-
.../triggers/DefaultTriggerStateMachine.java | 2 +-
.../triggers/ReshuffleTriggerStateMachine.java | 2 +-
.../core/triggers/TriggerStateMachine.java | 2 +-
.../TriggerStateMachineContextFactory.java | 18 +-
.../triggers/TriggerStateMachineRunner.java | 8 +-
runners/core-java/src/test/java/.placeholder | 0
.../core/DoFnDelegatingAggregatorTest.java | 144 -
...roupAlsoByWindowViaOutputBufferDoFnTest.java | 109 -
.../core/GroupAlsoByWindowsProperties.java | 752 -----
.../core/InMemoryStateInternalsTest.java | 60 +-
.../core/InMemoryTimerInternalsTest.java | 2 +-
.../runners/core/KeyedWorkItemCoderTest.java | 2 +-
.../core/LateDataDroppingDoFnRunnerTest.java | 50 +-
.../beam/runners/core/LateDataUtilsTest.java | 90 +
.../core/MergingActiveWindowSetTest.java | 2 +-
.../apache/beam/runners/core/NoOpOldDoFn.java | 72 -
.../beam/runners/core/OldDoFnContextTest.java | 72 -
.../apache/beam/runners/core/OldDoFnTest.java | 192 --
...ndedSplittableProcessElementInvokerTest.java | 1 -
.../core/PushbackSideInputDoFnRunnerTest.java | 282 --
.../beam/runners/core/ReduceFnRunnerTest.java | 130 +-
.../beam/runners/core/ReduceFnTester.java | 105 +-
.../beam/runners/core/SideInputHandlerTest.java | 2 +-
.../beam/runners/core/SimpleDoFnRunnerTest.java | 26 +-
.../runners/core/SimpleOldDoFnRunnerTest.java | 86 -
.../SimplePushbackSideInputDoFnRunnerTest.java | 281 ++
.../beam/runners/core/SplittableParDoTest.java | 97 +-
.../apache/beam/runners/core/StateTagTest.java | 83 +-
.../runners/core/StatefulDoFnRunnerTest.java | 68 +-
.../beam/runners/core/TimerInternalsTest.java | 2 +-
.../runners/core/metrics/CounterCellTest.java | 56 +
.../runners/core/metrics/DirtyStateTest.java | 56 +
.../core/metrics/DistributionCellTest.java | 54 +
.../runners/core/metrics/GaugeCellTest.java | 51 +
.../core/metrics/MetricUpdateMatchers.java | 82 +
.../core/metrics/MetricsContainerImplTest.java | 130 +
.../metrics/MetricsContainerStepMapTest.java | 272 ++
.../runners/core/metrics/MetricsMapTest.java | 103 +
.../triggers/TriggerStateMachineTester.java | 6 +-
.../core/triggers/TriggerStateMachinesTest.java | 2 +-
runners/direct-java/pom.xml | 111 +-
.../direct/AbstractModelEnforcement.java | 1 -
.../runners/direct/AggregatorContainer.java | 200 --
.../direct/BoundedReadEvaluatorFactory.java | 19 +-
.../beam/runners/direct/BundleFactory.java | 3 +-
.../org/apache/beam/runners/direct/Clock.java | 2 +-
.../runners/direct/CloningBundleFactory.java | 2 -
.../beam/runners/direct/CommittedBundle.java | 82 +
.../beam/runners/direct/CommittedResult.java | 3 +-
.../beam/runners/direct/CompletionCallback.java | 9 +-
.../CopyOnAccessInMemoryStateInternals.java | 216 +-
.../runners/direct/DirectExecutionContext.java | 15 +-
...ectGBKIntoKeyedWorkItemsOverrideFactory.java | 2 +-
.../apache/beam/runners/direct/DirectGraph.java | 14 +-
.../beam/runners/direct/DirectGraphVisitor.java | 2 +-
.../beam/runners/direct/DirectGroupByKey.java | 2 +-
.../direct/DirectGroupByKeyOverrideFactory.java | 2 +-
.../beam/runners/direct/DirectMetrics.java | 25 +-
.../beam/runners/direct/DirectOptions.java | 2 +-
.../beam/runners/direct/DirectRegistrar.java | 12 +-
.../beam/runners/direct/DirectRunner.java | 203 +-
.../runners/direct/DirectTimerInternals.java | 2 +-
.../runners/direct/DisplayDataValidator.java | 6 +-
.../runners/direct/DoFnLifecycleManager.java | 2 +-
...ecycleManagerRemovingTransformEvaluator.java | 6 +-
.../beam/runners/direct/EmptyInputProvider.java | 3 +-
.../runners/direct/EmptyTransformEvaluator.java | 50 -
.../beam/runners/direct/EvaluationContext.java | 45 +-
.../runners/direct/ExecutorServiceFactory.java | 2 +-
.../direct/ExecutorServiceParallelExecutor.java | 42 +-
.../runners/direct/FlattenEvaluatorFactory.java | 4 +-
.../GroupAlsoByWindowEvaluatorFactory.java | 42 +-
.../direct/GroupByKeyOnlyEvaluatorFactory.java | 4 +-
.../ImmutabilityCheckingBundleFactory.java | 2 -
.../direct/ImmutabilityEnforcementFactory.java | 3 +-
.../direct/ImmutableListBundleFactory.java | 2 -
.../beam/runners/direct/ModelEnforcement.java | 12 +-
.../runners/direct/ModelEnforcementFactory.java | 5 +-
.../beam/runners/direct/NanosOffsetClock.java | 2 +-
.../runners/direct/PCollectionViewWindow.java | 67 +
.../runners/direct/PCollectionViewWriter.java | 34 +
.../beam/runners/direct/ParDoEvaluator.java | 136 +-
.../runners/direct/ParDoEvaluatorFactory.java | 16 +-
.../direct/ParDoMultiOverrideFactory.java | 13 +-
.../direct/PassthroughTransformEvaluator.java | 3 +-
.../beam/runners/direct/PipelineExecutor.java | 3 +-
.../beam/runners/direct/RootInputProvider.java | 3 +-
.../runners/direct/RootProviderRegistry.java | 3 +-
.../beam/runners/direct/SideInputContainer.java | 7 +-
...littableProcessElementsEvaluatorFactory.java | 120 +-
.../direct/StatefulParDoEvaluatorFactory.java | 17 +-
.../apache/beam/runners/direct/StepAndKey.java | 2 +-
.../runners/direct/StepTransformResult.java | 19 +-
.../direct/TestStreamEvaluatorFactory.java | 6 +-
.../beam/runners/direct/TransformEvaluator.java | 3 +-
.../direct/TransformEvaluatorFactory.java | 8 +-
.../direct/TransformEvaluatorRegistry.java | 7 +-
.../beam/runners/direct/TransformExecutor.java | 25 +-
.../beam/runners/direct/TransformResult.java | 15 +-
.../direct/UnboundedReadEvaluatorFactory.java | 6 +-
.../beam/runners/direct/UncommittedBundle.java | 57 +
.../runners/direct/ViewEvaluatorFactory.java | 5 +-
.../runners/direct/ViewOverrideFactory.java | 2 +-
.../direct/WatermarkCallbackExecutor.java | 4 +-
.../beam/runners/direct/WatermarkManager.java | 7 +-
.../runners/direct/WindowEvaluatorFactory.java | 4 +-
.../direct/WriteWithShardingFactory.java | 17 +-
.../runners/direct/AggregatorContainerTest.java | 137 -
.../direct/BoundedReadEvaluatorFactoryTest.java | 8 +-
.../direct/CloningBundleFactoryTest.java | 22 +-
.../runners/direct/CommittedResultTest.java | 16 +-
.../CopyOnAccessInMemoryStateInternalsTest.java | 196 +-
.../runners/direct/DirectGraphVisitorTest.java | 8 +-
.../beam/runners/direct/DirectGraphs.java | 2 +-
.../DirectGroupByKeyOverrideFactoryTest.java | 2 +-
.../beam/runners/direct/DirectMetricsTest.java | 15 +-
.../direct/DirectRunnerApiSurfaceTest.java | 68 +
.../beam/runners/direct/DirectRunnerTest.java | 20 +-
.../direct/DirectTimerInternalsTest.java | 2 +-
...leManagerRemovingTransformEvaluatorTest.java | 10 +-
.../runners/direct/EvaluationContextTest.java | 62 +-
.../direct/FlattenEvaluatorFactoryTest.java | 4 +-
.../direct/GroupByKeyEvaluatorFactoryTest.java | 2 -
.../GroupByKeyOnlyEvaluatorFactoryTest.java | 2 -
.../ImmutabilityCheckingBundleFactoryTest.java | 2 -
.../ImmutabilityEnforcementFactoryTest.java | 3 +-
.../direct/ImmutableListBundleFactoryTest.java | 2 -
.../direct/KeyedPValueTrackingVisitorTest.java | 2 +-
.../apache/beam/runners/direct/MockClock.java | 2 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 17 +-
.../runners/direct/SideInputContainerTest.java | 8 +-
.../StatefulParDoEvaluatorFactoryTest.java | 24 +-
.../runners/direct/StepTransformResultTest.java | 3 +-
.../direct/TestStreamEvaluatorFactoryTest.java | 3 +-
.../runners/direct/TransformExecutorTest.java | 8 +-
.../UnboundedReadEvaluatorFactoryTest.java | 11 +-
.../direct/ViewEvaluatorFactoryTest.java | 6 +-
.../runners/direct/ViewOverrideFactoryTest.java | 6 +-
.../direct/WatermarkCallbackExecutorTest.java | 4 +-
.../runners/direct/WatermarkManagerTest.java | 6 +-
.../direct/WindowEvaluatorFactoryTest.java | 2 -
.../direct/WriteWithShardingFactoryTest.java | 67 +-
runners/flink/examples/pom.xml | 126 -
.../beam/runners/flink/examples/TFIDF.java | 455 ---
.../beam/runners/flink/examples/WordCount.java | 129 -
.../runners/flink/examples/package-info.java | 22 -
.../flink/examples/streaming/AutoComplete.java | 400 ---
.../flink/examples/streaming/JoinExamples.java | 154 -
.../examples/streaming/WindowedWordCount.java | 141 -
.../flink/examples/streaming/package-info.java | 22 -
runners/flink/pom.xml | 291 +-
runners/flink/runner/pom.xml | 330 ---
.../flink/DefaultParallelismFactory.java | 39 -
.../flink/FlinkBatchPipelineTranslator.java | 139 -
.../flink/FlinkBatchTransformTranslators.java | 723 -----
.../flink/FlinkBatchTranslationContext.java | 153 -
.../flink/FlinkDetachedRunnerResult.java | 75 -
.../FlinkPipelineExecutionEnvironment.java | 241 --
.../runners/flink/FlinkPipelineOptions.java | 101 -
.../runners/flink/FlinkPipelineTranslator.java | 53 -
.../apache/beam/runners/flink/FlinkRunner.java | 232 --
.../runners/flink/FlinkRunnerRegistrar.java | 62 -
.../beam/runners/flink/FlinkRunnerResult.java | 98 -
.../flink/FlinkStreamingPipelineTranslator.java | 276 --
.../FlinkStreamingTransformTranslators.java | 1044 -------
.../flink/FlinkStreamingTranslationContext.java | 130 -
.../flink/FlinkStreamingViewOverrides.java | 372 ---
.../flink/PipelineTranslationOptimizer.java | 72 -
.../beam/runners/flink/TestFlinkRunner.java | 84 -
.../beam/runners/flink/TranslationMode.java | 31 -
.../apache/beam/runners/flink/package-info.java | 22 -
.../functions/FlinkAggregatorFactory.java | 53 -
.../functions/FlinkAssignContext.java | 63 -
.../functions/FlinkAssignWindows.java | 49 -
.../functions/FlinkDoFnFunction.java | 161 -
.../FlinkMergingNonShuffleReduceFunction.java | 228 --
.../FlinkMergingPartialReduceFunction.java | 201 --
.../functions/FlinkMergingReduceFunction.java | 199 --
.../FlinkMultiOutputPruningFunction.java | 50 -
.../functions/FlinkNoOpStepContext.java | 73 -
.../functions/FlinkPartialReduceFunction.java | 172 --
.../functions/FlinkReduceFunction.java | 173 --
.../functions/FlinkSideInputReader.java | 80 -
.../functions/FlinkStatefulDoFnFunction.java | 198 --
.../functions/SideInputInitializer.java | 73 -
.../translation/functions/package-info.java | 22 -
.../runners/flink/translation/package-info.java | 22 -
.../translation/types/CoderTypeInformation.java | 120 -
.../translation/types/CoderTypeSerializer.java | 132 -
.../types/EncodedValueComparator.java | 195 --
.../types/EncodedValueSerializer.java | 113 -
.../types/EncodedValueTypeInformation.java | 98 -
.../types/InspectableByteArrayOutputStream.java | 34 -
.../flink/translation/types/KvKeySelector.java | 50 -
.../flink/translation/types/package-info.java | 22 -
.../utils/SerializedPipelineOptions.java | 67 -
.../flink/translation/utils/package-info.java | 22 -
.../wrappers/DataInputViewWrapper.java | 58 -
.../wrappers/DataOutputViewWrapper.java | 51 -
.../SerializableFnAggregatorWrapper.java | 98 -
.../translation/wrappers/SourceInputFormat.java | 149 -
.../translation/wrappers/SourceInputSplit.java | 52 -
.../translation/wrappers/package-info.java | 22 -
.../wrappers/streaming/DoFnOperator.java | 772 -----
.../streaming/KvToByteBufferKeySelector.java | 56 -
.../streaming/SingletonKeyedWorkItem.java | 56 -
.../streaming/SingletonKeyedWorkItemCoder.java | 126 -
.../streaming/SplittableDoFnOperator.java | 150 -
.../wrappers/streaming/WindowDoFnOperator.java | 117 -
.../wrappers/streaming/WorkItemKeySelector.java | 56 -
.../streaming/io/BoundedSourceWrapper.java | 218 --
.../streaming/io/UnboundedSocketSource.java | 249 --
.../streaming/io/UnboundedSourceWrapper.java | 476 ---
.../wrappers/streaming/io/package-info.java | 22 -
.../wrappers/streaming/package-info.java | 22 -
.../state/FlinkBroadcastStateInternals.java | 865 ------
.../state/FlinkKeyGroupStateInternals.java | 487 ----
.../state/FlinkSplitStateInternals.java | 260 --
.../streaming/state/FlinkStateInternals.java | 1053 -------
.../state/KeyGroupCheckpointedOperator.java | 35 -
.../state/KeyGroupRestoringOperator.java | 32 -
.../wrappers/streaming/state/package-info.java | 22 -
.../runner/src/main/resources/log4j.properties | 23 -
.../flink/EncodedValueComparatorTest.java | 70 -
.../runners/flink/FlinkRunnerRegistrarTest.java | 48 -
.../beam/runners/flink/FlinkTestPipeline.java | 72 -
.../beam/runners/flink/PipelineOptionsTest.java | 184 --
.../beam/runners/flink/ReadSourceITCase.java | 85 -
.../flink/ReadSourceStreamingITCase.java | 74 -
.../beam/runners/flink/WriteSinkITCase.java | 192 --
.../flink/streaming/DoFnOperatorTest.java | 600 ----
.../FlinkBroadcastStateInternalsTest.java | 245 --
.../FlinkKeyGroupStateInternalsTest.java | 262 --
.../streaming/FlinkSplitStateInternalsTest.java | 101 -
.../streaming/FlinkStateInternalsTest.java | 395 ---
.../flink/streaming/GroupByNullKeyTest.java | 124 -
.../flink/streaming/TestCountingSource.java | 254 --
.../streaming/TopWikipediaSessionsITCase.java | 133 -
.../streaming/UnboundedSourceWrapperTest.java | 464 ---
.../runners/flink/streaming/package-info.java | 22 -
.../src/test/resources/log4j-test.properties | 27 -
.../flink/DefaultParallelismFactory.java | 39 +
.../flink/FlinkBatchPipelineTranslator.java | 139 +
.../flink/FlinkBatchTransformTranslators.java | 685 +++++
.../flink/FlinkBatchTranslationContext.java | 157 +
.../flink/FlinkDetachedRunnerResult.java | 63 +
.../FlinkPipelineExecutionEnvironment.java | 249 ++
.../runners/flink/FlinkPipelineOptions.java | 119 +
.../runners/flink/FlinkPipelineTranslator.java | 53 +
.../apache/beam/runners/flink/FlinkRunner.java | 234 ++
.../runners/flink/FlinkRunnerRegistrar.java | 62 +
.../beam/runners/flink/FlinkRunnerResult.java | 81 +
.../flink/FlinkStreamingPipelineTranslator.java | 279 ++
.../FlinkStreamingTransformTranslators.java | 1078 +++++++
.../flink/FlinkStreamingTranslationContext.java | 133 +
.../flink/FlinkStreamingViewOverrides.java | 372 +++
.../flink/PipelineTranslationOptimizer.java | 72 +
.../beam/runners/flink/TestFlinkRunner.java | 84 +
.../beam/runners/flink/TranslationMode.java | 31 +
.../metrics/DoFnRunnerWithMetricsUpdate.java | 94 +
.../flink/metrics/FlinkMetricContainer.java | 199 ++
.../flink/metrics/MetricsAccumulator.java | 60 +
.../flink/metrics/ReaderInvocationUtil.java | 75 +
.../runners/flink/metrics/package-info.java | 22 +
.../apache/beam/runners/flink/package-info.java | 22 +
.../functions/AbstractFlinkCombineRunner.java | 182 ++
.../functions/FlinkAssignContext.java | 63 +
.../functions/FlinkAssignWindows.java | 49 +
.../functions/FlinkDoFnFunction.java | 170 ++
.../FlinkMergingNonShuffleReduceFunction.java | 93 +
.../FlinkMultiOutputPruningFunction.java | 50 +
.../functions/FlinkNoOpStepContext.java | 73 +
.../functions/FlinkPartialReduceFunction.java | 94 +
.../functions/FlinkReduceFunction.java | 95 +
.../functions/FlinkSideInputReader.java | 80 +
.../functions/FlinkStatefulDoFnFunction.java | 207 ++
.../functions/HashingFlinkCombineRunner.java | 180 ++
.../functions/SideInputInitializer.java | 73 +
.../functions/SortingFlinkCombineRunner.java | 192 ++
.../translation/functions/package-info.java | 22 +
.../runners/flink/translation/package-info.java | 22 +
.../translation/types/CoderTypeInformation.java | 120 +
.../translation/types/CoderTypeSerializer.java | 132 +
.../types/EncodedValueComparator.java | 195 ++
.../types/EncodedValueSerializer.java | 111 +
.../types/EncodedValueTypeInformation.java | 98 +
.../types/InspectableByteArrayOutputStream.java | 34 +
.../flink/translation/types/KvKeySelector.java | 50 +
.../flink/translation/types/package-info.java | 22 +
.../utils/SerializedPipelineOptions.java | 77 +
.../flink/translation/utils/package-info.java | 22 +
.../wrappers/DataInputViewWrapper.java | 58 +
.../wrappers/DataOutputViewWrapper.java | 51 +
.../translation/wrappers/SourceInputFormat.java | 165 ++
.../translation/wrappers/SourceInputSplit.java | 52 +
.../translation/wrappers/package-info.java | 22 +
.../wrappers/streaming/DoFnOperator.java | 797 +++++
.../streaming/KvToByteBufferKeySelector.java | 56 +
.../streaming/SingletonKeyedWorkItem.java | 56 +
.../streaming/SingletonKeyedWorkItemCoder.java | 120 +
.../streaming/SplittableDoFnOperator.java | 178 ++
.../wrappers/streaming/WindowDoFnOperator.java | 119 +
.../wrappers/streaming/WorkItemKeySelector.java | 56 +
.../streaming/io/BoundedSourceWrapper.java | 231 ++
.../wrappers/streaming/io/DedupingOperator.java | 187 ++
.../streaming/io/UnboundedSocketSource.java | 249 ++
.../streaming/io/UnboundedSourceWrapper.java | 492 ++++
.../wrappers/streaming/io/package-info.java | 22 +
.../wrappers/streaming/package-info.java | 22 +
.../state/FlinkBroadcastStateInternals.java | 837 ++++++
.../state/FlinkKeyGroupStateInternals.java | 472 +++
.../state/FlinkSplitStateInternals.java | 246 ++
.../streaming/state/FlinkStateInternals.java | 1028 +++++++
.../state/KeyGroupCheckpointedOperator.java | 35 +
.../state/KeyGroupRestoringOperator.java | 32 +
.../wrappers/streaming/state/package-info.java | 22 +
.../flink/src/main/resources/log4j.properties | 23 +
.../flink/EncodedValueComparatorTest.java | 70 +
.../runners/flink/FlinkRunnerRegistrarTest.java | 48 +
.../beam/runners/flink/FlinkTestPipeline.java | 69 +
.../beam/runners/flink/PipelineOptionsTest.java | 290 ++
.../beam/runners/flink/ReadSourceITCase.java | 85 +
.../flink/ReadSourceStreamingITCase.java | 74 +
.../flink/streaming/DedupingOperatorTest.java | 131 +
.../flink/streaming/DoFnOperatorTest.java | 605 ++++
.../FlinkBroadcastStateInternalsTest.java | 245 ++
.../FlinkKeyGroupStateInternalsTest.java | 262 ++
.../streaming/FlinkSplitStateInternalsTest.java | 101 +
.../streaming/FlinkStateInternalsTest.java | 393 +++
.../flink/streaming/GroupByNullKeyTest.java | 129 +
.../flink/streaming/TestCountingSource.java | 254 ++
.../streaming/TopWikipediaSessionsITCase.java | 133 +
.../streaming/UnboundedSourceWrapperTest.java | 465 +++
.../runners/flink/streaming/package-info.java | 22 +
.../src/test/resources/log4j-test.properties | 27 +
runners/google-cloud-dataflow-java/pom.xml | 139 +-
.../beam/runners/dataflow/AssignWindows.java | 7 +-
.../dataflow/BatchStatefulParDoOverrides.java | 4 +-
.../runners/dataflow/BatchViewOverrides.java | 49 +-
.../dataflow/DataflowAggregatorTransforms.java | 79 -
.../dataflow/DataflowMetricUpdateExtractor.java | 109 -
.../beam/runners/dataflow/DataflowMetrics.java | 55 +-
.../runners/dataflow/DataflowPipelineJob.java | 121 +-
.../dataflow/DataflowPipelineRegistrar.java | 4 +-
.../dataflow/DataflowPipelineTranslator.java | 53 +-
.../beam/runners/dataflow/DataflowRunner.java | 230 +-
.../dataflow/PrimitiveParDoSingleFactory.java | 2 +-
.../beam/runners/dataflow/ReadTranslator.java | 22 +-
.../dataflow/ReshuffleOverrideFactory.java | 8 +-
.../dataflow/StreamingViewOverrides.java | 2 +-
.../dataflow/TestDataflowPipelineOptions.java | 28 +
.../runners/dataflow/TestDataflowRunner.java | 388 +++
.../runners/dataflow/TransformTranslator.java | 2 +-
.../dataflow/internal/CustomSources.java | 20 +-
.../runners/dataflow/internal/IsmFormat.java | 175 +-
.../options/DataflowPipelineOptions.java | 26 +-
.../testing/TestDataflowPipelineOptions.java | 28 -
.../dataflow/testing/TestDataflowRunner.java | 359 ---
.../runners/dataflow/testing/package-info.java | 24 -
.../util/AvroCoderCloudObjectTranslator.java | 60 +
.../runners/dataflow/util/CloudKnownType.java | 138 +
.../beam/runners/dataflow/util/CloudObject.java | 184 ++
.../runners/dataflow/util/CloudObjectKinds.java | 31 +
.../dataflow/util/CloudObjectTranslator.java | 46 +
.../dataflow/util/CloudObjectTranslators.java | 584 ++++
.../runners/dataflow/util/CloudObjects.java | 91 +
.../CoderCloudObjectTranslatorRegistrar.java | 46 +
.../dataflow/util/DataflowTemplateJob.java | 3 +-
.../dataflow/util/DataflowTransport.java | 2 +-
...aultCoderCloudObjectTranslatorRegistrar.java | 134 +
.../beam/runners/dataflow/util/DoFnInfo.java | 2 +-
.../beam/runners/dataflow/util/GcsStager.java | 26 +-
.../runners/dataflow/util/MonitoringUtil.java | 4 +-
.../beam/runners/dataflow/util/PackageUtil.java | 41 +-
.../runners/dataflow/util/PropertyNames.java | 112 +
.../runners/dataflow/util/RandomAccessData.java | 27 +-
.../SerializableCoderCloudObjectTranslator.java | 63 +
.../beam/runners/dataflow/util/Structs.java | 372 +++
.../BatchStatefulParDoOverridesTest.java | 10 +-
.../runners/dataflow/DataflowMetricsTest.java | 11 +-
.../dataflow/DataflowPipelineJobTest.java | 510 +---
.../dataflow/DataflowPipelineRegistrarTest.java | 2 +-
.../DataflowPipelineTranslatorTest.java | 249 +-
.../runners/dataflow/DataflowRunnerTest.java | 139 +-
.../PrimitiveParDoSingleFactoryTest.java | 2 +-
.../dataflow/TestDataflowRunnerTest.java | 634 ++++
.../options/DataflowPipelineOptionsTest.java | 21 +-
.../options/DataflowProfilingOptionsTest.java | 4 +-
.../DataflowWorkerLoggingOptionsTest.java | 4 +-
.../testing/TestDataflowRunnerTest.java | 775 -----
.../transforms/DataflowGroupByKeyTest.java | 4 +-
.../dataflow/transforms/DataflowViewTest.java | 4 +-
.../runners/dataflow/util/CloudObjectsTest.java | 216 ++
.../dataflow/util/MonitoringUtilTest.java | 2 +-
.../runners/dataflow/util/PackageUtilTest.java | 147 +-
.../dataflow/util/RandomAccessDataTest.java | 6 +-
.../beam/runners/dataflow/util/StructsTest.java | 206 ++
runners/pom.xml | 42 +-
runners/spark/README.md | 124 -
runners/spark/pom.xml | 72 +-
.../spark/SparkNativePipelineVisitor.java | 2 +-
.../beam/runners/spark/SparkPipelineResult.java | 20 +-
.../apache/beam/runners/spark/SparkRunner.java | 8 +-
.../beam/runners/spark/SparkRunnerDebugger.java | 7 +-
.../runners/spark/SparkRunnerRegistrar.java | 2 +-
.../beam/runners/spark/TestSparkRunner.java | 72 +-
.../spark/aggregators/NamedAggregators.java | 6 +-
.../spark/aggregators/SparkAggregators.java | 110 -
.../beam/runners/spark/coders/CoderHelpers.java | 1 -
.../spark/coders/StatelessJavaSerializer.java | 1 -
.../beam/runners/spark/examples/WordCount.java | 17 +-
.../beam/runners/spark/io/CreateStream.java | 2 +-
.../beam/runners/spark/io/MicrobatchSource.java | 193 +-
.../beam/runners/spark/io/SourceDStream.java | 57 +-
.../apache/beam/runners/spark/io/SourceRDD.java | 190 +-
.../runners/spark/io/SparkUnboundedSource.java | 20 +-
.../runners/spark/metrics/AggregatorMetric.java | 1 -
.../spark/metrics/MetricsAccumulator.java | 20 +-
.../spark/metrics/MetricsAccumulatorParam.java | 20 +-
.../runners/spark/metrics/SparkBeamMetric.java | 11 +-
.../spark/metrics/SparkBeamMetricSource.java | 3 +-
.../spark/metrics/SparkMetricResults.java | 172 --
.../spark/metrics/SparkMetricsContainer.java | 174 --
.../SparkGroupAlsoByWindowViaWindowSet.java | 60 +-
.../spark/stateful/SparkStateInternals.java | 131 +-
.../spark/stateful/SparkTimerInternals.java | 2 +-
.../spark/stateful/StateSpecFunctions.java | 39 +-
.../translation/DoFnRunnerWithMetrics.java | 11 +-
.../spark/translation/EvaluationContext.java | 18 +-
.../translation/GroupCombineFunctions.java | 5 +-
.../spark/translation/MultiDoFnFunction.java | 10 +-
.../translation/SparkAbstractCombineFn.java | 8 +-
.../spark/translation/SparkAssignWindowFn.java | 3 +-
.../spark/translation/SparkGlobalCombineFn.java | 39 +-
...SparkGroupAlsoByWindowViaOutputBufferFn.java | 15 +-
.../spark/translation/SparkKeyedCombineFn.java | 65 +-
.../spark/translation/SparkProcessContext.java | 2 +-
.../spark/translation/SparkRuntimeContext.java | 108 +-
.../translation/StorageLevelPTransform.java | 2 +-
.../spark/translation/TransformTranslator.java | 52 +-
.../spark/translation/TranslationUtils.java | 5 +-
.../SparkRunnerStreamingContextFactory.java | 2 +-
.../streaming/StreamingTransformTranslator.java | 106 +-
.../spark/util/GlobalWatermarkHolder.java | 1 -
.../util/SinglePrimitiveOutputPTransform.java | 2 +-
.../spark/util/SparkSideInputReader.java | 4 +-
.../apache/beam/runners/spark/CacheTest.java | 14 +-
.../beam/runners/spark/ForceStreamingTest.java | 18 +-
.../apache/beam/runners/spark/PipelineRule.java | 109 -
.../runners/spark/ProvidedSparkContextTest.java | 10 +-
.../runners/spark/SparkRunnerDebuggerTest.java | 36 +-
.../beam/runners/spark/StreamingTest.java | 23 +
.../TestSparkPipelineOptionsRegistrar.java | 36 +
.../spark/aggregators/ClearAggregatorsRule.java | 38 -
.../metrics/sink/NamedAggregatorsTest.java | 101 -
.../metrics/sink/SparkMetricsSinkTest.java | 78 +
.../beam/runners/spark/io/AvroPipelineTest.java | 14 +-
.../beam/runners/spark/io/NumShardsTest.java | 9 +-
.../spark/io/ReaderToIteratorAdapterTest.java | 145 +
.../translation/SparkRuntimeContextTest.java | 122 +
.../spark/translation/StorageLevelTest.java | 31 +-
.../translation/streaming/CreateStreamTest.java | 99 +-
.../ResumeFromCheckpointStreamingTest.java | 150 +-
.../streaming/StreamingSourceMetricsTest.java | 35 +-
.../streaming/TrackStreamingSourcesTest.java | 4 +-
sdks/common/fn-api/pom.xml | 2 +-
sdks/common/pom.xml | 2 +-
sdks/common/runner-api/pom.xml | 5 +-
sdks/java/build-tools/pom.xml | 2 +-
.../src/main/resources/beam/findbugs-filter.xml | 29 +-
sdks/java/core/pom.xml | 217 +-
.../beam/sdk/AggregatorPipelineExtractor.java | 92 -
.../beam/sdk/AggregatorRetrievalException.java | 33 -
.../org/apache/beam/sdk/AggregatorValues.java | 51 -
.../main/java/org/apache/beam/sdk/Pipeline.java | 224 +-
.../org/apache/beam/sdk/PipelineResult.java | 21 +-
.../org/apache/beam/sdk/PipelineRunner.java | 61 +
.../beam/sdk/annotations/Experimental.java | 38 +-
.../apache/beam/sdk/annotations/Internal.java | 42 +
.../org/apache/beam/sdk/coders/AtomicCoder.java | 63 +-
.../org/apache/beam/sdk/coders/AvroCoder.java | 218 +-
.../apache/beam/sdk/coders/BigDecimalCoder.java | 39 +-
.../beam/sdk/coders/BigEndianIntegerCoder.java | 13 +-
.../beam/sdk/coders/BigEndianLongCoder.java | 13 +-
.../apache/beam/sdk/coders/BigIntegerCoder.java | 29 +-
.../org/apache/beam/sdk/coders/BitSetCoder.java | 74 +
.../apache/beam/sdk/coders/ByteArrayCoder.java | 27 +-
.../org/apache/beam/sdk/coders/ByteCoder.java | 10 +-
.../apache/beam/sdk/coders/ByteStringCoder.java | 113 -
.../sdk/coders/CannotProvideCoderException.java | 2 +-
.../java/org/apache/beam/sdk/coders/Coder.java | 204 +-
.../apache/beam/sdk/coders/CoderFactories.java | 274 --
.../apache/beam/sdk/coders/CoderFactory.java | 44 -
.../apache/beam/sdk/coders/CoderProvider.java | 19 +-
.../beam/sdk/coders/CoderProviderRegistrar.java | 42 +
.../apache/beam/sdk/coders/CoderProviders.java | 240 +-
.../apache/beam/sdk/coders/CoderRegistry.java | 616 ++--
.../apache/beam/sdk/coders/CollectionCoder.java | 22 -
.../org/apache/beam/sdk/coders/CustomCoder.java | 84 +-
.../apache/beam/sdk/coders/DefaultCoder.java | 119 +-
.../apache/beam/sdk/coders/DelegateCoder.java | 54 +-
.../sdk/coders/DeterministicStandardCoder.java | 39 -
.../org/apache/beam/sdk/coders/DoubleCoder.java | 10 +-
.../apache/beam/sdk/coders/DurationCoder.java | 25 +-
.../apache/beam/sdk/coders/InstantCoder.java | 30 +-
.../apache/beam/sdk/coders/IterableCoder.java | 31 -
.../beam/sdk/coders/IterableLikeCoder.java | 37 +-
.../org/apache/beam/sdk/coders/JAXBCoder.java | 201 --
.../org/apache/beam/sdk/coders/KvCoder.java | 64 +-
.../beam/sdk/coders/LengthPrefixCoder.java | 51 +-
.../org/apache/beam/sdk/coders/ListCoder.java | 24 +-
.../org/apache/beam/sdk/coders/MapCoder.java | 60 +-
.../apache/beam/sdk/coders/NullableCoder.java | 58 +-
.../beam/sdk/coders/SerializableCoder.java | 94 +-
.../org/apache/beam/sdk/coders/SetCoder.java | 25 -
.../apache/beam/sdk/coders/StandardCoder.java | 265 --
.../beam/sdk/coders/StringDelegateCoder.java | 33 +-
.../apache/beam/sdk/coders/StringUtf8Coder.java | 32 +-
.../beam/sdk/coders/StructuralByteArray.java | 4 +-
.../apache/beam/sdk/coders/StructuredCoder.java | 134 +
.../beam/sdk/coders/TableRowJsonCoder.java | 88 -
.../beam/sdk/coders/TextualIntegerCoder.java | 22 +-
.../org/apache/beam/sdk/coders/VarIntCoder.java | 13 +-
.../apache/beam/sdk/coders/VarLongCoder.java | 23 +-
.../org/apache/beam/sdk/coders/VoidCoder.java | 13 +-
.../apache/beam/sdk/coders/package-info.java | 2 +-
.../beam/sdk/coders/protobuf/ProtoCoder.java | 405 ---
.../beam/sdk/coders/protobuf/ProtobufUtil.java | 171 --
.../beam/sdk/coders/protobuf/package-info.java | 24 -
.../java/org/apache/beam/sdk/io/AvroIO.java | 1323 +++------
.../java/org/apache/beam/sdk/io/AvroSink.java | 132 +
.../java/org/apache/beam/sdk/io/AvroSource.java | 176 +-
.../apache/beam/sdk/io/BlockBasedSource.java | 10 +-
.../sdk/io/BoundedReadFromUnboundedSource.java | 8 +-
.../org/apache/beam/sdk/io/BoundedSource.java | 4 +-
.../apache/beam/sdk/io/CompressedSource.java | 36 +-
.../org/apache/beam/sdk/io/CountingInput.java | 283 --
.../org/apache/beam/sdk/io/CountingSource.java | 55 +-
.../beam/sdk/io/DefaultFilenamePolicy.java | 169 ++
.../org/apache/beam/sdk/io/FileBasedSink.java | 862 +++---
.../org/apache/beam/sdk/io/FileBasedSource.java | 381 +--
.../java/org/apache/beam/sdk/io/FileSystem.java | 22 +
.../apache/beam/sdk/io/FileSystemRegistrar.java | 18 +-
.../org/apache/beam/sdk/io/FileSystems.java | 235 +-
.../apache/beam/sdk/io/GenerateSequence.java | 194 ++
.../org/apache/beam/sdk/io/LocalFileSystem.java | 39 +-
.../beam/sdk/io/LocalFileSystemRegistrar.java | 18 +-
.../org/apache/beam/sdk/io/LocalResourceId.java | 45 +-
.../org/apache/beam/sdk/io/LocalResources.java | 59 +
.../apache/beam/sdk/io/OffsetBasedSource.java | 4 +-
.../main/java/org/apache/beam/sdk/io/Read.java | 8 +-
.../apache/beam/sdk/io/ShardNameTemplate.java | 2 +-
.../main/java/org/apache/beam/sdk/io/Sink.java | 295 --
.../java/org/apache/beam/sdk/io/TFRecordIO.java | 734 ++---
.../java/org/apache/beam/sdk/io/TextIO.java | 1295 +++------
.../java/org/apache/beam/sdk/io/TextSink.java | 127 +
.../java/org/apache/beam/sdk/io/TextSource.java | 236 ++
.../org/apache/beam/sdk/io/UnboundedSource.java | 25 +-
.../main/java/org/apache/beam/sdk/io/Write.java | 603 ----
.../java/org/apache/beam/sdk/io/WriteFiles.java | 599 ++++
.../java/org/apache/beam/sdk/io/XmlSink.java | 321 --
.../java/org/apache/beam/sdk/io/XmlSource.java | 551 ----
.../org/apache/beam/sdk/io/fs/MatchResult.java | 66 +-
.../org/apache/beam/sdk/io/fs/ResourceId.java | 39 +-
.../org/apache/beam/sdk/io/package-info.java | 2 +-
.../org/apache/beam/sdk/io/range/ByteKey.java | 21 +-
.../apache/beam/sdk/metrics/CounterCell.java | 84 -
.../org/apache/beam/sdk/metrics/DirtyState.java | 98 -
.../beam/sdk/metrics/DistributionCell.java | 70 -
.../beam/sdk/metrics/DistributionData.java | 60 -
.../beam/sdk/metrics/DistributionResult.java | 3 +
.../org/apache/beam/sdk/metrics/GaugeCell.java | 60 -
.../org/apache/beam/sdk/metrics/GaugeData.java | 81 -
.../apache/beam/sdk/metrics/GaugeResult.java | 3 +
.../org/apache/beam/sdk/metrics/Metric.java | 13 +-
.../org/apache/beam/sdk/metrics/MetricCell.java | 47 -
.../beam/sdk/metrics/MetricFiltering.java | 99 -
.../org/apache/beam/sdk/metrics/MetricKey.java | 41 -
.../apache/beam/sdk/metrics/MetricResults.java | 29 +-
.../apache/beam/sdk/metrics/MetricUpdates.java | 78 -
.../org/apache/beam/sdk/metrics/Metrics.java | 45 +-
.../beam/sdk/metrics/MetricsContainer.java | 129 +-
.../beam/sdk/metrics/MetricsEnvironment.java | 14 +-
.../apache/beam/sdk/metrics/MetricsFilter.java | 9 +-
.../org/apache/beam/sdk/metrics/MetricsMap.java | 86 -
.../apache/beam/sdk/metrics/SinkMetrics.java | 53 +
.../apache/beam/sdk/metrics/SourceMetrics.java | 119 +
.../beam/sdk/options/BigQueryOptions.java | 32 -
.../options/CloudResourceManagerOptions.java | 40 -
.../DefaultPipelineOptionsRegistrar.java | 8 +-
.../org/apache/beam/sdk/options/GcpOptions.java | 227 --
.../org/apache/beam/sdk/options/GcsOptions.java | 158 -
.../beam/sdk/options/GoogleApiDebugOptions.java | 87 -
.../beam/sdk/options/PipelineOptionSpec.java | 4 +-
.../beam/sdk/options/PipelineOptions.java | 15 +-
.../sdk/options/PipelineOptionsFactory.java | 6 +-
.../sdk/options/ProxyInvocationHandler.java | 16 +-
.../apache/beam/sdk/options/PubsubOptions.java | 36 -
.../apache/beam/sdk/options/ValueProvider.java | 15 +-
.../beam/sdk/options/ValueProviderUtils.java | 60 -
.../apache/beam/sdk/options/ValueProviders.java | 61 +
.../java/org/apache/beam/sdk/package-info.java | 2 +-
.../beam/sdk/runners/AppliedPTransform.java | 73 +
.../beam/sdk/runners/PTransformMatcher.java | 7 +-
.../beam/sdk/runners/PTransformOverride.java | 6 +-
.../sdk/runners/PTransformOverrideFactory.java | 8 +-
.../apache/beam/sdk/runners/PipelineRunner.java | 63 -
.../sdk/runners/PipelineRunnerRegistrar.java | 5 +
.../beam/sdk/runners/TransformHierarchy.java | 15 +-
.../apache/beam/sdk/runners/package-info.java | 4 +-
.../org/apache/beam/sdk/state/BagState.java | 36 +
.../apache/beam/sdk/state/CombiningState.java | 55 +
.../apache/beam/sdk/state/GroupingState.java | 44 +
.../org/apache/beam/sdk/state/MapState.java | 73 +
.../apache/beam/sdk/state/ReadableState.java | 51 +
.../apache/beam/sdk/state/ReadableStates.java | 44 +
.../org/apache/beam/sdk/state/SetState.java | 47 +
.../java/org/apache/beam/sdk/state/State.java | 36 +
.../org/apache/beam/sdk/state/StateBinder.java | 70 +
.../org/apache/beam/sdk/state/StateContext.java | 46 +
.../apache/beam/sdk/state/StateContexts.java | 51 +
.../org/apache/beam/sdk/state/StateSpec.java | 63 +
.../org/apache/beam/sdk/state/StateSpecs.java | 683 +++++
.../org/apache/beam/sdk/state/TimeDomain.java | 49 +
.../java/org/apache/beam/sdk/state/Timer.java | 70 +
.../org/apache/beam/sdk/state/TimerSpec.java | 30 +
.../org/apache/beam/sdk/state/TimerSpecs.java | 39 +
.../java/org/apache/beam/sdk/state/Timers.java | 58 +
.../org/apache/beam/sdk/state/ValueState.java | 35 +
.../beam/sdk/state/WatermarkHoldState.java | 41 +
.../org/apache/beam/sdk/state/package-info.java | 22 +
.../beam/sdk/testing/BigqueryMatcher.java | 256 --
.../beam/sdk/testing/CoderProperties.java | 53 +-
.../apache/beam/sdk/testing/CrashingRunner.java | 2 +-
.../beam/sdk/testing/FileChecksumMatcher.java | 2 +-
.../beam/sdk/testing/MatcherDeserializer.java | 4 +-
.../beam/sdk/testing/MatcherSerializer.java | 4 +-
.../apache/beam/sdk/testing/NeedsRunner.java | 2 +-
.../org/apache/beam/sdk/testing/PAssert.java | 168 +-
.../beam/sdk/testing/SourceTestUtils.java | 11 +-
.../beam/sdk/testing/SuccessOrFailure.java | 82 +
.../apache/beam/sdk/testing/TestPipeline.java | 93 +-
.../beam/sdk/testing/TestPipelineOptions.java | 2 -
.../org/apache/beam/sdk/testing/TestStream.java | 120 +-
.../beam/sdk/testing/UsesCounterMetrics.java | 25 +
.../sdk/testing/UsesDistributionMetrics.java | 26 +
.../beam/sdk/testing/UsesGaugeMetrics.java | 25 +
.../apache/beam/sdk/testing/UsesMapState.java | 2 +-
.../apache/beam/sdk/testing/UsesSetState.java | 2 +-
...esSplittableParDoWithWindowedSideInputs.java | 26 +
.../beam/sdk/testing/WindowFnTestUtils.java | 53 +-
.../apache/beam/sdk/transforms/Aggregator.java | 70 -
.../sdk/transforms/AggregatorRetriever.java | 45 -
.../beam/sdk/transforms/AppliedPTransform.java | 73 -
.../sdk/transforms/ApproximateQuantiles.java | 98 +-
.../beam/sdk/transforms/ApproximateUnique.java | 3 +-
.../org/apache/beam/sdk/transforms/Combine.java | 728 ++---
.../beam/sdk/transforms/CombineFnBase.java | 160 +-
.../apache/beam/sdk/transforms/CombineFns.java | 480 +--
.../beam/sdk/transforms/CombineWithContext.java | 177 +-
.../org/apache/beam/sdk/transforms/Count.java | 47 +-
.../org/apache/beam/sdk/transforms/Create.java | 76 +-
.../sdk/transforms/DelegatingAggregator.java | 126 -
.../org/apache/beam/sdk/transforms/DoFn.java | 219 +-
.../apache/beam/sdk/transforms/DoFnTester.java | 138 +-
.../org/apache/beam/sdk/transforms/Filter.java | 35 +-
.../org/apache/beam/sdk/transforms/Flatten.java | 2 +-
.../apache/beam/sdk/transforms/GroupByKey.java | 32 +-
.../beam/sdk/transforms/GroupIntoBatches.java | 77 +-
.../org/apache/beam/sdk/transforms/Latest.java | 20 +-
.../beam/sdk/transforms/Materialization.java | 8 +-
.../beam/sdk/transforms/Materializations.java | 11 +-
.../org/apache/beam/sdk/transforms/Mean.java | 23 +-
.../apache/beam/sdk/transforms/PTransform.java | 18 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 26 +-
.../org/apache/beam/sdk/transforms/Regex.java | 1 -
.../beam/sdk/transforms/ReifyTimestamps.java | 79 +
.../apache/beam/sdk/transforms/Reshuffle.java | 97 +
.../org/apache/beam/sdk/transforms/Top.java | 52 +-
.../org/apache/beam/sdk/transforms/View.java | 67 +-
.../org/apache/beam/sdk/transforms/ViewFn.java | 6 +-
.../apache/beam/sdk/transforms/WithKeys.java | 4 +-
.../sdk/transforms/display/DisplayData.java | 6 +
.../beam/sdk/transforms/join/CoGbkResult.java | 53 +-
.../sdk/transforms/join/CoGbkResultSchema.java | 21 +-
.../beam/sdk/transforms/join/UnionCoder.java | 43 +-
.../reflect/ByteBuddyDoFnInvokerFactory.java | 31 +-
.../reflect/ByteBuddyOnTimerInvokerFactory.java | 20 +-
.../sdk/transforms/reflect/DoFnInvoker.java | 23 +-
.../sdk/transforms/reflect/DoFnSignature.java | 50 +-
.../sdk/transforms/reflect/DoFnSignatures.java | 93 +-
.../reflect/StableInvokerNamingStrategy.java | 54 +
.../sdk/transforms/reflect/package-info.java | 5 +-
.../beam/sdk/transforms/windowing/AfterAll.java | 8 +-
.../sdk/transforms/windowing/AfterEach.java | 2 +-
.../sdk/transforms/windowing/AfterFirst.java | 5 +-
.../sdk/transforms/windowing/AfterPane.java | 5 +-
.../windowing/AfterProcessingTime.java | 6 +-
.../transforms/windowing/AfterWatermark.java | 9 +-
.../transforms/windowing/DefaultTrigger.java | 10 +-
.../sdk/transforms/windowing/GlobalWindow.java | 23 +-
.../sdk/transforms/windowing/GlobalWindows.java | 5 +-
.../transforms/windowing/IntervalWindow.java | 36 +-
.../transforms/windowing/InvalidWindows.java | 2 +-
.../MergeOverlappingIntervalWindows.java | 6 +-
.../beam/sdk/transforms/windowing/Never.java | 4 +-
.../windowing/NonMergingWindowFn.java | 2 +-
.../transforms/windowing/OrFinallyTrigger.java | 7 +-
.../sdk/transforms/windowing/OutputTimeFn.java | 314 --
.../sdk/transforms/windowing/OutputTimeFns.java | 212 --
.../beam/sdk/transforms/windowing/PaneInfo.java | 19 +-
.../sdk/transforms/windowing/Repeatedly.java | 6 +-
.../transforms/windowing/ReshuffleTrigger.java | 58 +
.../beam/sdk/transforms/windowing/Sessions.java | 13 +-
.../transforms/windowing/TimestampCombiner.java | 191 ++
.../windowing/TimestampTransform.java | 41 +-
.../beam/sdk/transforms/windowing/Trigger.java | 30 +-
.../beam/sdk/transforms/windowing/Window.java | 44 +-
.../beam/sdk/transforms/windowing/WindowFn.java | 10 +-
.../transforms/windowing/WindowMappingFn.java | 12 +-
.../org/apache/beam/sdk/util/ApiSurface.java | 14 +-
.../beam/sdk/util/AppEngineEnvironment.java | 62 -
.../apache/beam/sdk/util/AppliedCombineFn.java | 36 +-
...AttemptAndTimeBoundedExponentialBackOff.java | 170 --
.../util/AttemptBoundedExponentialBackOff.java | 86 -
.../org/apache/beam/sdk/util/AvroUtils.java | 137 -
.../java/org/apache/beam/sdk/util/BackOff.java | 81 +
.../org/apache/beam/sdk/util/BackOffUtils.java | 57 +
.../org/apache/beam/sdk/util/BitSetCoder.java | 24 +-
.../org/apache/beam/sdk/util/ClassPath.java | 545 ++++
.../apache/beam/sdk/util/CloudKnownType.java | 138 -
.../org/apache/beam/sdk/util/CloudObject.java | 184 --
.../org/apache/beam/sdk/util/CoderUtils.java | 142 +-
.../beam/sdk/util/CombineContextFactory.java | 27 +-
.../org/apache/beam/sdk/util/CombineFnUtil.java | 125 +-
.../apache/beam/sdk/util/CredentialFactory.java | 29 -
.../org/apache/beam/sdk/util/DefaultBucket.java | 105 -
.../beam/sdk/util/ExplicitShardedFile.java | 26 +-
.../beam/sdk/util/FileIOChannelFactory.java | 223 --
.../sdk/util/FileIOChannelFactoryRegistrar.java | 38 -
.../org/apache/beam/sdk/util/FluentBackoff.java | 1 -
.../beam/sdk/util/GcpCredentialFactory.java | 67 -
.../apache/beam/sdk/util/GcpProjectUtil.java | 106 -
.../beam/sdk/util/GcsIOChannelFactory.java | 111 -
.../sdk/util/GcsIOChannelFactoryRegistrar.java | 38 -
.../apache/beam/sdk/util/GcsPathValidator.java | 95 -
.../java/org/apache/beam/sdk/util/GcsUtil.java | 798 -----
.../apache/beam/sdk/util/IOChannelFactory.java | 127 -
.../sdk/util/IOChannelFactoryRegistrar.java | 48 -
.../apache/beam/sdk/util/IOChannelUtils.java | 328 ---
.../apache/beam/sdk/util/IdentityWindowFn.java | 1 +
.../util/IntervalBoundedExponentialBackOff.java | 89 -
.../beam/sdk/util/NoopCredentialFactory.java | 68 -
.../apache/beam/sdk/util/NoopPathValidator.java | 49 -
.../sdk/util/NullCredentialInitializer.java | 62 -
.../beam/sdk/util/NullSideInputReader.java | 61 -
.../beam/sdk/util/NumberedShardedFile.java | 32 +-
.../beam/sdk/util/PCollectionViewWindow.java | 67 -
.../apache/beam/sdk/util/PCollectionViews.java | 494 ----
.../org/apache/beam/sdk/util/PathValidator.java | 48 -
.../org/apache/beam/sdk/util/PropertyNames.java | 112 -
.../sdk/util/ReadyCheckingSideInputReader.java | 34 -
.../apache/beam/sdk/util/ReifyTimestamps.java | 76 -
.../org/apache/beam/sdk/util/ReleaseInfo.java | 70 +-
.../org/apache/beam/sdk/util/Reshuffle.java | 87 -
.../apache/beam/sdk/util/ReshuffleTrigger.java | 52 -
.../sdk/util/RetryHttpRequestInitializer.java | 248 --
.../apache/beam/sdk/util/SerializableUtils.java | 26 +-
.../org/apache/beam/sdk/util/Serializer.java | 144 -
.../org/apache/beam/sdk/util/ShardedFile.java | 2 -
.../apache/beam/sdk/util/SideInputReader.java | 47 -
.../java/org/apache/beam/sdk/util/Sleeper.java | 48 +
.../java/org/apache/beam/sdk/util/Structs.java | 383 ---
.../beam/sdk/util/SystemDoFnInternal.java | 3 -
.../apache/beam/sdk/util/TestCredential.java | 59 -
.../org/apache/beam/sdk/util/TimeDomain.java | 43 -
.../java/org/apache/beam/sdk/util/Timer.java | 67 -
.../org/apache/beam/sdk/util/TimerSpec.java | 30 -
.../org/apache/beam/sdk/util/TimerSpecs.java | 41 -
.../java/org/apache/beam/sdk/util/Timers.java | 58 -
.../org/apache/beam/sdk/util/Transport.java | 178 --
.../sdk/util/UploadIdResponseInterceptor.java | 60 -
.../apache/beam/sdk/util/ValueWithRecordId.java | 146 -
.../java/org/apache/beam/sdk/util/Values.java | 88 -
.../org/apache/beam/sdk/util/WindowedValue.java | 112 +-
.../apache/beam/sdk/util/WindowingStrategy.java | 371 ---
.../org/apache/beam/sdk/util/gcsfs/GcsPath.java | 626 ----
.../beam/sdk/util/gcsfs/package-info.java | 20 -
.../apache/beam/sdk/util/state/BagState.java | 28 -
.../beam/sdk/util/state/CombiningState.java | 53 -
.../beam/sdk/util/state/GroupingState.java | 42 -
.../apache/beam/sdk/util/state/MapState.java | 79 -
.../beam/sdk/util/state/ReadableState.java | 51 -
.../beam/sdk/util/state/ReadableStates.java | 45 -
.../apache/beam/sdk/util/state/SetState.java | 45 -
.../org/apache/beam/sdk/util/state/State.java | 32 -
.../apache/beam/sdk/util/state/StateBinder.java | 73 -
.../beam/sdk/util/state/StateContext.java | 43 -
.../beam/sdk/util/state/StateContexts.java | 51 -
.../apache/beam/sdk/util/state/StateSpec.java | 54 -
.../apache/beam/sdk/util/state/StateSpecs.java | 709 -----
.../apache/beam/sdk/util/state/ValueState.java | 37 -
.../beam/sdk/util/state/WatermarkHoldState.java | 43 -
.../beam/sdk/util/state/package-info.java | 22 -
.../java/org/apache/beam/sdk/values/PBegin.java | 26 +-
.../org/apache/beam/sdk/values/PCollection.java | 171 +-
.../apache/beam/sdk/values/PCollectionList.java | 14 +-
.../beam/sdk/values/PCollectionTuple.java | 24 +-
.../apache/beam/sdk/values/PCollectionView.java | 33 +-
.../beam/sdk/values/PCollectionViews.java | 495 ++++
.../java/org/apache/beam/sdk/values/PDone.java | 27 +-
.../org/apache/beam/sdk/values/POutput.java | 25 +-
.../beam/sdk/values/POutputValueBase.java | 74 -
.../java/org/apache/beam/sdk/values/PValue.java | 12 +-
.../org/apache/beam/sdk/values/PValueBase.java | 76 +-
.../apache/beam/sdk/values/TaggedPValue.java | 6 +-
.../beam/sdk/values/TimestampedValue.java | 48 +-
.../org/apache/beam/sdk/values/TupleTag.java | 22 -
.../org/apache/beam/sdk/values/TypedPValue.java | 208 --
.../beam/sdk/values/ValueInSingleWindow.java | 48 +-
.../beam/sdk/values/ValueWithRecordId.java | 149 +
.../beam/sdk/values/WindowingStrategy.java | 283 ++
sdks/java/core/src/main/proto/README.md | 45 -
.../main/proto/proto2_coder_test_messages.proto | 53 -
.../org/apache/beam/SdkCoreApiSurfaceTest.java | 7 -
.../sdk/AggregatorPipelineExtractorTest.java | 226 --
.../java/org/apache/beam/sdk/PipelineTest.java | 100 +-
.../apache/beam/sdk/coders/AvroCoderTest.java | 93 +-
.../beam/sdk/coders/AvroCoderTestPojo.java | 51 +
.../beam/sdk/coders/BigDecimalCoderTest.java | 23 +-
.../sdk/coders/BigEndianIntegerCoderTest.java | 8 -
.../beam/sdk/coders/BigEndianLongCoderTest.java | 8 -
.../beam/sdk/coders/BigIntegerCoderTest.java | 23 +-
.../apache/beam/sdk/coders/BitSetCoderTest.java | 103 +
.../beam/sdk/coders/ByteArrayCoderTest.java | 8 -
.../apache/beam/sdk/coders/ByteCoderTest.java | 8 -
.../beam/sdk/coders/ByteStringCoderTest.java | 128 -
.../beam/sdk/coders/CoderFactoriesTest.java | 100 -
.../beam/sdk/coders/CoderProvidersTest.java | 82 +-
.../beam/sdk/coders/CoderRegistryTest.java | 269 +-
.../beam/sdk/coders/CollectionCoderTest.java | 8 -
.../apache/beam/sdk/coders/CustomCoderTest.java | 50 +-
.../beam/sdk/coders/DefaultCoderTest.java | 71 +-
.../beam/sdk/coders/DelegateCoderTest.java | 61 -
.../apache/beam/sdk/coders/DoubleCoderTest.java | 8 -
.../beam/sdk/coders/InstantCoderTest.java | 8 -
.../beam/sdk/coders/IterableCoderTest.java | 35 -
.../apache/beam/sdk/coders/JAXBCoderTest.java | 223 --
.../org/apache/beam/sdk/coders/KvCoderTest.java | 20 -
.../beam/sdk/coders/LengthPrefixCoderTest.java | 42 +-
.../apache/beam/sdk/coders/ListCoderTest.java | 25 -
.../apache/beam/sdk/coders/MapCoderTest.java | 28 -
.../beam/sdk/coders/NullableCoderTest.java | 42 +-
.../beam/sdk/coders/SerializableCoderTest.java | 64 +-
.../apache/beam/sdk/coders/SetCoderTest.java | 8 -
.../beam/sdk/coders/StandardCoderTest.java | 238 --
.../sdk/coders/StringDelegateCoderTest.java | 6 -
.../beam/sdk/coders/StructuredCoderTest.java | 238 ++
.../beam/sdk/coders/TableRowJsonCoderTest.java | 94 -
.../sdk/coders/TextualIntegerCoderTest.java | 8 -
.../apache/beam/sdk/coders/VarIntCoderTest.java | 8 -
.../beam/sdk/coders/VarLongCoderTest.java | 8 -
.../sdk/coders/protobuf/ProtoCoderTest.java | 182 --
.../sdk/coders/protobuf/ProtobufUtilTest.java | 192 --
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 243 +-
.../apache/beam/sdk/io/AvroIOTransformTest.java | 30 +-
.../org/apache/beam/sdk/io/AvroSourceTest.java | 66 +-
.../beam/sdk/io/CompressedSourceTest.java | 37 +-
.../apache/beam/sdk/io/CountingInputTest.java | 221 --
.../apache/beam/sdk/io/CountingSourceTest.java | 10 +-
.../beam/sdk/io/DefaultFilenamePolicyTest.java | 55 +
.../sdk/io/DrunkWritableByteChannelFactory.java | 1 -
.../apache/beam/sdk/io/FileBasedSinkTest.java | 512 ++--
.../apache/beam/sdk/io/FileBasedSourceTest.java | 198 +-
.../org/apache/beam/sdk/io/FileSystemsTest.java | 20 +-
.../beam/sdk/io/GenerateSequenceTest.java | 194 ++
.../sdk/io/LocalFileSystemRegistrarTest.java | 6 +
.../apache/beam/sdk/io/LocalFileSystemTest.java | 91 +-
.../apache/beam/sdk/io/LocalResourceIdTest.java | 51 +
.../beam/sdk/io/OffsetBasedSourceTest.java | 8 +-
.../java/org/apache/beam/sdk/io/ReadTest.java | 4 +-
.../java/org/apache/beam/sdk/io/SimpleSink.java | 97 +
.../org/apache/beam/sdk/io/TFRecordIOTest.java | 28 +-
.../java/org/apache/beam/sdk/io/TextIOTest.java | 264 +-
.../org/apache/beam/sdk/io/WriteFilesTest.java | 468 +++
.../java/org/apache/beam/sdk/io/WriteTest.java | 705 -----
.../org/apache/beam/sdk/io/XmlSinkTest.java | 260 --
.../org/apache/beam/sdk/io/XmlSourceTest.java | 846 ------
.../apache/beam/sdk/io/fs/ResourceIdTester.java | 150 +
.../range/ByteKeyRangeInterpolateKeyTest.java | 1 -
.../beam/sdk/io/range/ByteKeyRangeTest.java | 16 +-
.../apache/beam/sdk/io/range/ByteKeyTest.java | 4 +-
.../beam/sdk/metrics/CounterCellTest.java | 55 -
.../apache/beam/sdk/metrics/DirtyStateTest.java | 56 -
.../beam/sdk/metrics/DistributionCellTest.java | 53 -
.../apache/beam/sdk/metrics/GaugeCellTest.java | 48 -
.../beam/sdk/metrics/MetricFilteringTest.java | 145 -
.../apache/beam/sdk/metrics/MetricMatchers.java | 296 --
.../beam/sdk/metrics/MetricResultsMatchers.java | 190 ++
.../beam/sdk/metrics/MetricsContainerTest.java | 129 -
.../sdk/metrics/MetricsEnvironmentTest.java | 23 +-
.../apache/beam/sdk/metrics/MetricsMapTest.java | 103 -
.../apache/beam/sdk/metrics/MetricsTest.java | 226 +-
.../apache/beam/sdk/options/GcpOptionsTest.java | 171 --
.../sdk/options/GoogleApiDebugOptionsTest.java | 145 -
.../sdk/options/PipelineOptionsFactoryTest.java | 77 +-
.../beam/sdk/options/PipelineOptionsTest.java | 13 +
.../sdk/options/ProxyInvocationHandlerTest.java | 6 +-
.../beam/sdk/options/ValueProviderTest.java | 23 +-
.../sdk/options/ValueProviderUtilsTest.java | 78 -
.../beam/sdk/options/ValueProvidersTest.java | 79 +
.../beam/sdk/runners/PipelineRunnerTest.java | 47 +-
.../sdk/runners/TransformHierarchyTest.java | 12 +-
.../beam/sdk/runners/TransformTreeTest.java | 14 +-
.../runners/dataflow/TestCountingSource.java | 2 +-
.../beam/sdk/testing/BigqueryMatcherTest.java | 176 --
.../beam/sdk/testing/CoderPropertiesTest.java | 73 +-
.../beam/sdk/testing/CrashingRunnerTest.java | 2 +-
.../beam/sdk/testing/ExpectedLogsTest.java | 2 +-
.../sdk/testing/FastNanoClockAndSleeper.java | 47 -
.../testing/FastNanoClockAndSleeperTest.java | 47 -
.../sdk/testing/FileChecksumMatcherTest.java | 12 +-
.../beam/sdk/testing/GatherAllPanesTest.java | 8 +-
.../apache/beam/sdk/testing/PAssertTest.java | 77 +-
.../sdk/testing/PCollectionViewTesting.java | 2 +-
.../sdk/testing/SerializableMatchersTest.java | 5 +-
.../beam/sdk/testing/SourceTestUtilsTest.java | 2 +-
.../beam/sdk/testing/SystemNanoTimeSleeper.java | 3 +-
.../beam/sdk/testing/TestPipelineTest.java | 12 +-
.../apache/beam/sdk/testing/TestStreamTest.java | 24 -
.../beam/sdk/testing/WindowSupplierTest.java | 8 +-
.../sdk/transforms/ApproximateUniqueTest.java | 2 +-
.../beam/sdk/transforms/CombineFnsTest.java | 144 +-
.../apache/beam/sdk/transforms/CombineTest.java | 257 +-
.../apache/beam/sdk/transforms/CreateTest.java | 34 +-
.../apache/beam/sdk/transforms/DoFnTest.java | 162 --
.../beam/sdk/transforms/DoFnTesterTest.java | 61 +-
.../apache/beam/sdk/transforms/FilterTest.java | 13 +
.../sdk/transforms/FlatMapElementsTest.java | 4 +-
.../apache/beam/sdk/transforms/FlattenTest.java | 8 +-
.../beam/sdk/transforms/GroupByKeyTest.java | 26 +-
.../beam/sdk/transforms/LatestFnTest.java | 44 +-
.../beam/sdk/transforms/MapElementsTest.java | 4 +-
.../beam/sdk/transforms/ParDoLifecycleTest.java | 16 +-
.../apache/beam/sdk/transforms/ParDoTest.java | 441 +--
.../apache/beam/sdk/transforms/RegexTest.java | 2 -
.../sdk/transforms/ReifyTimestampsTest.java | 140 +
.../beam/sdk/transforms/ReshuffleTest.java | 293 ++
.../beam/sdk/transforms/SplittableDoFnTest.java | 108 +-
.../org/apache/beam/sdk/transforms/SumTest.java | 2 +-
.../apache/beam/sdk/transforms/ViewTest.java | 19 +-
.../display/DisplayDataEvaluator.java | 2 +-
.../display/DisplayDataEvaluatorTest.java | 5 +-
.../sdk/transforms/display/DisplayDataTest.java | 17 +
.../sdk/transforms/join/CoGroupByKeyTest.java | 6 +-
.../sdk/transforms/join/UnionCoderTest.java | 11 +
.../transforms/reflect/DoFnInvokersTest.java | 79 +-
.../DoFnSignaturesSplittableDoFnTest.java | 4 +-
.../transforms/reflect/DoFnSignaturesTest.java | 61 +-
.../transforms/reflect/OnTimerInvokersTest.java | 38 +-
.../testhelper/DoFnInvokersTestHelper.java | 6 +-
.../transforms/windowing/GlobalWindowTest.java | 9 +-
.../transforms/windowing/OutputTimeFnsTest.java | 51 -
.../windowing/ReshuffleTriggerTest.java | 42 +
.../sdk/transforms/windowing/SessionsTest.java | 6 +-
.../sdk/transforms/windowing/WindowTest.java | 31 +-
.../sdk/transforms/windowing/WindowingTest.java | 4 +-
...mptAndTimeBoundedExponentialBackOffTest.java | 213 --
.../AttemptBoundedExponentialBackOffTest.java | 85 -
.../org/apache/beam/sdk/util/AvroUtilsTest.java | 147 -
...BufferedElementCountingOutputStreamTest.java | 5 +-
.../apache/beam/sdk/util/CoderUtilsTest.java | 108 +-
.../apache/beam/sdk/util/CombineFnUtilTest.java | 20 +-
.../apache/beam/sdk/util/DefaultBucketTest.java | 112 -
.../util/FileIOChannelFactoryRegistrarTest.java | 44 -
.../beam/sdk/util/FileIOChannelFactoryTest.java | 242 --
.../apache/beam/sdk/util/FluentBackoffTest.java | 1 -
.../beam/sdk/util/GcpProjectUtilTest.java | 76 -
.../util/GcsIOChannelFactoryRegistrarTest.java | 44 -
.../beam/sdk/util/GcsIOChannelFactoryTest.java | 43 -
.../beam/sdk/util/GcsPathValidatorTest.java | 87 -
.../org/apache/beam/sdk/util/GcsUtilTest.java | 798 -----
.../beam/sdk/util/IOChannelUtilsTest.java | 148 -
.../IntervalBoundedExponentialBackOffTest.java | 100 -
.../org/apache/beam/sdk/util/NameUtilsTest.java | 2 +-
.../beam/sdk/util/NumberedShardedFileTest.java | 55 +-
.../beam/sdk/util/ReifyTimestampsTest.java | 109 -
.../org/apache/beam/sdk/util/ReshuffleTest.java | 270 --
.../beam/sdk/util/ReshuffleTriggerTest.java | 44 -
.../util/RetryHttpRequestInitializerTest.java | 290 --
.../beam/sdk/util/SerializableUtilsTest.java | 60 +-
.../apache/beam/sdk/util/SerializerTest.java | 162 --
.../org/apache/beam/sdk/util/StructsTest.java | 206 --
.../util/UploadIdResponseInterceptorTest.java | 98 -
.../beam/sdk/util/ValueWithRecordIdTest.java | 34 -
.../apache/beam/sdk/util/WindowedValueTest.java | 10 -
.../apache/beam/sdk/util/gcsfs/GcsPathTest.java | 358 ---
.../beam/sdk/values/PCollectionListTest.java | 25 +-
.../beam/sdk/values/PCollectionTupleTest.java | 7 +-
.../org/apache/beam/sdk/values/PDoneTest.java | 2 +-
.../apache/beam/sdk/values/TypedPValueTest.java | 28 +-
.../beam/sdk/values/ValueWithRecordIdTest.java | 34 +
.../google-cloud-platform-core/pom.xml | 191 ++
.../extensions/gcp/auth/CredentialFactory.java | 29 +
.../gcp/auth/GcpCredentialFactory.java | 67 +
.../gcp/auth/NoopCredentialFactory.java | 68 +
.../gcp/auth/NullCredentialInitializer.java | 62 +
.../sdk/extensions/gcp/auth/package-info.java | 22 +
.../options/CloudResourceManagerOptions.java | 32 +
.../sdk/extensions/gcp/options/GcpOptions.java | 390 +++
.../options/GcpPipelineOptionsRegistrar.java | 39 +
.../sdk/extensions/gcp/options/GcsOptions.java | 160 +
.../gcp/options/GoogleApiDebugOptions.java | 89 +
.../extensions/gcp/options/package-info.java | 22 +
.../gcp/storage/GcsCreateOptions.java | 56 +
.../extensions/gcp/storage/GcsFileSystem.java | 270 ++
.../gcp/storage/GcsFileSystemRegistrar.java | 46 +
.../gcp/storage/GcsPathValidator.java | 105 +
.../extensions/gcp/storage/GcsResourceId.java | 128 +
.../gcp/storage/NoopPathValidator.java | 53 +
.../extensions/gcp/storage/PathValidator.java | 59 +
.../extensions/gcp/storage/package-info.java | 21 +
.../apache/beam/sdk/util/BackOffAdapter.java | 43 +
.../java/org/apache/beam/sdk/util/GcsUtil.java | 785 +++++
.../sdk/util/RetryHttpRequestInitializer.java | 192 ++
.../org/apache/beam/sdk/util/Transport.java | 122 +
.../sdk/util/UploadIdResponseInterceptor.java | 60 +
.../org/apache/beam/sdk/util/gcsfs/GcsPath.java | 627 ++++
.../beam/sdk/util/gcsfs/package-info.java | 20 +
.../org/apache/beam/sdk/util/package-info.java | 20 +
.../extensions/gcp/GcpCoreApiSurfaceTest.java | 59 +
.../sdk/extensions/gcp/auth/TestCredential.java | 59 +
.../extensions/gcp/options/GcpOptionsTest.java | 273 ++
.../gcp/options/GoogleApiDebugOptionsTest.java | 149 +
.../gcp/storage/GcsFileSystemRegistrarTest.java | 52 +
.../gcp/storage/GcsFileSystemTest.java | 274 ++
.../gcp/storage/GcsPathValidatorTest.java | 107 +
.../gcp/storage/GcsResourceIdTest.java | 178 ++
.../beam/sdk/util/FastNanoClockAndSleeper.java | 47 +
.../sdk/util/FastNanoClockAndSleeperTest.java | 47 +
.../org/apache/beam/sdk/util/GcsUtilTest.java | 841 ++++++
.../util/RetryHttpRequestInitializerTest.java | 281 ++
.../util/UploadIdResponseInterceptorTest.java | 98 +
.../apache/beam/sdk/util/gcsfs/GcsPathTest.java | 358 +++
sdks/java/extensions/jackson/pom.xml | 99 +-
sdks/java/extensions/join-library/README.md | 42 -
sdks/java/extensions/join-library/pom.xml | 24 +-
sdks/java/extensions/pom.xml | 4 +-
sdks/java/extensions/protobuf/pom.xml | 121 +
.../extensions/protobuf/ByteStringCoder.java | 124 +
.../sdk/extensions/protobuf/ProtoCoder.java | 349 +++
.../ProtobufCoderProviderRegistrar.java | 41 +
.../sdk/extensions/protobuf/ProtobufUtil.java | 171 ++
.../sdk/extensions/protobuf/package-info.java | 24 +
.../protobuf/ByteStringCoderTest.java | 129 +
.../sdk/extensions/protobuf/ProtoCoderTest.java | 172 ++
.../extensions/protobuf/ProtobufUtilTest.java | 191 ++
.../test/proto/proto2_coder_test_messages.proto | 53 +
sdks/java/extensions/sorter/README.md | 42 -
sdks/java/extensions/sorter/pom.xml | 26 +-
.../extensions/sorter/ExternalSorterTest.java | 1 -
sdks/java/harness/pom.xml | 7 +-
.../org/apache/beam/fn/harness/FnHarness.java | 10 +-
.../harness/control/ProcessBundleHandler.java | 6 +-
.../BeamFnDataBufferingOutboundObserver.java | 3 +-
.../harness/data/BeamFnDataInboundObserver.java | 3 +-
.../fn/harness/fake/FakeAggregatorFactory.java | 52 -
.../beam/fn/harness/fake/FakeStepContext.java | 2 +-
.../fn/harness/logging/BeamFnLoggingClient.java | 2 +-
.../harness/stream/StreamObserverFactory.java | 2 +-
.../beam/runners/core/BeamFnDataReadRunner.java | 21 +-
.../runners/core/BeamFnDataWriteRunner.java | 21 +-
.../apache/beam/fn/harness/FnHarnessTest.java | 2 +-
.../control/ProcessBundleHandlerTest.java | 33 +-
...BeamFnDataBufferingOutboundObserverTest.java | 3 +-
.../data/BeamFnDataInboundObserverTest.java | 3 +-
.../runners/core/BeamFnDataReadRunnerTest.java | 4 +-
.../runners/core/BeamFnDataWriteRunnerTest.java | 4 +-
sdks/java/io/common/pom.xml | 2 +-
.../apache/beam/sdk/io/common/HashingFn.java | 2 -
sdks/java/io/elasticsearch/pom.xml | 3 +-
.../sdk/io/elasticsearch/ElasticsearchIO.java | 18 +-
.../elasticsearch/ElasticSearchIOTestUtils.java | 2 +-
.../sdk/io/elasticsearch/ElasticsearchIOIT.java | 2 +-
.../io/elasticsearch/ElasticsearchIOTest.java | 4 +-
sdks/java/io/google-cloud-platform/pom.xml | 61 +-
.../sdk/io/gcp/bigquery/BatchLoadBigQuery.java | 180 --
.../beam/sdk/io/gcp/bigquery/BatchLoads.java | 338 +++
.../BigQueryCoderProviderRegistrar.java | 40 +
.../sdk/io/gcp/bigquery/BigQueryHelpers.java | 152 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 626 ++--
.../sdk/io/gcp/bigquery/BigQueryOptions.java | 39 +
.../io/gcp/bigquery/BigQueryQuerySource.java | 45 +-
.../sdk/io/gcp/bigquery/BigQueryServices.java | 1 -
.../io/gcp/bigquery/BigQueryServicesImpl.java | 144 +-
.../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 79 +-
.../gcp/bigquery/BigQueryTableRowIterator.java | 7 +-
.../io/gcp/bigquery/BigQueryTableSource.java | 22 +-
.../sdk/io/gcp/bigquery/CalculateSchemas.java | 78 +
.../beam/sdk/io/gcp/bigquery/CreateTables.java | 146 +
.../io/gcp/bigquery/DynamicDestinations.java | 178 ++
.../bigquery/DynamicDestinationsHelpers.java | 193 ++
.../io/gcp/bigquery/GenerateShardedTable.java | 47 +
.../io/gcp/bigquery/PassThroughThenCleanup.java | 4 +-
.../beam/sdk/io/gcp/bigquery/PrepareWrite.java | 66 +
.../beam/sdk/io/gcp/bigquery/ShardedKey.java | 25 +-
.../sdk/io/gcp/bigquery/ShardedKeyCoder.java | 33 +-
.../sdk/io/gcp/bigquery/StreamWithDeDup.java | 90 -
.../sdk/io/gcp/bigquery/StreamingInserts.java | 66 +
.../sdk/io/gcp/bigquery/StreamingWriteFn.java | 90 +-
.../io/gcp/bigquery/StreamingWriteTables.java | 93 +
.../sdk/io/gcp/bigquery/TableDestination.java | 82 +
.../io/gcp/bigquery/TableDestinationCoder.java | 59 +
.../sdk/io/gcp/bigquery/TableRowInfoCoder.java | 18 +-
.../sdk/io/gcp/bigquery/TableRowJsonCoder.java | 100 +
.../sdk/io/gcp/bigquery/TableRowWriter.java | 74 +-
.../sdk/io/gcp/bigquery/TagWithUniqueIds.java | 62 +
.../gcp/bigquery/TagWithUniqueIdsAndTable.java | 135 -
.../sdk/io/gcp/bigquery/TransformingSource.java | 4 +-
.../beam/sdk/io/gcp/bigquery/WriteBundles.java | 82 -
.../io/gcp/bigquery/WriteBundlesToFiles.java | 240 ++
.../bigquery/WriteGroupedRecordsToFiles.java | 68 +
.../sdk/io/gcp/bigquery/WritePartition.java | 165 +-
.../beam/sdk/io/gcp/bigquery/WriteRename.java | 82 +-
.../beam/sdk/io/gcp/bigquery/WriteResult.java | 20 +-
.../beam/sdk/io/gcp/bigquery/WriteTables.java | 183 +-
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 271 +-
.../io/gcp/bigtable/BigtableServiceImpl.java | 13 +-
.../common/GcpIoPipelineOptionsRegistrar.java | 39 +
.../beam/sdk/io/gcp/common/package-info.java | 20 +
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 25 +-
.../beam/sdk/io/gcp/pubsub/PubsubClient.java | 43 +-
.../pubsub/PubsubCoderProviderRegistrar.java | 37 +
.../sdk/io/gcp/pubsub/PubsubGrpcClient.java | 39 +-
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 688 +++--
.../sdk/io/gcp/pubsub/PubsubJsonClient.java | 37 +-
.../beam/sdk/io/gcp/pubsub/PubsubMessage.java | 61 +
.../pubsub/PubsubMessagePayloadOnlyCoder.java | 58 +
.../PubsubMessageWithAttributesCoder.java | 71 +
.../beam/sdk/io/gcp/pubsub/PubsubOptions.java | 44 +
.../sdk/io/gcp/pubsub/PubsubTestClient.java | 7 +-
.../sdk/io/gcp/pubsub/PubsubUnboundedSink.java | 186 +-
.../io/gcp/pubsub/PubsubUnboundedSource.java | 232 +-
.../beam/sdk/io/gcp/storage/GcsFileSystem.java | 242 --
.../io/gcp/storage/GcsFileSystemRegistrar.java | 49 -
.../beam/sdk/io/gcp/storage/GcsResourceId.java | 116 -
.../beam/sdk/io/gcp/storage/package-info.java | 21 -
.../beam/sdk/io/gcp/testing/package-info.java | 21 +
.../beam/sdk/io/gcp/GcpApiSurfaceTest.java | 2 +
.../BigQueryCoderProviderRegistrarTest.java | 40 +
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 1788 +++++-------
.../gcp/bigquery/BigQueryServicesImplTest.java | 24 +-
.../io/gcp/bigquery/FakeBigQueryServices.java | 162 ++
.../sdk/io/gcp/bigquery/FakeDatasetService.java | 208 ++
.../sdk/io/gcp/bigquery/FakeJobService.java | 412 +++
.../sdk/io/gcp/bigquery/TableContainer.java | 60 +
.../io/gcp/bigquery/TableRowJsonCoderTest.java | 87 +
.../sdk/io/gcp/bigtable/BigtableIOTest.java | 32 +-
.../sdk/io/gcp/bigtable/BigtableWriteIT.java | 6 +-
.../sdk/io/gcp/datastore/DatastoreV1Test.java | 4 +-
.../beam/sdk/io/gcp/datastore/V1ReadIT.java | 2 +-
.../beam/sdk/io/gcp/datastore/V1TestUtil.java | 10 +-
.../beam/sdk/io/gcp/datastore/V1WriteIT.java | 6 +-
.../sdk/io/gcp/pubsub/PubsubClientTest.java | 50 +-
.../sdk/io/gcp/pubsub/PubsubGrpcClientTest.java | 18 +-
.../beam/sdk/io/gcp/pubsub/PubsubIOTest.java | 88 +-
.../sdk/io/gcp/pubsub/PubsubJsonClientTest.java | 14 +-
.../io/gcp/pubsub/PubsubUnboundedSinkTest.java | 89 +-
.../gcp/pubsub/PubsubUnboundedSourceTest.java | 114 +-
.../gcp/storage/GcsFileSystemRegistrarTest.java | 51 -
.../sdk/io/gcp/storage/GcsFileSystemTest.java | 274 --
.../sdk/io/gcp/storage/GcsResourceIdTest.java | 147 -
.../sdk/io/gcp/testing/BigqueryMatcher.java | 259 ++
.../sdk/io/gcp/testing/BigqueryMatcherTest.java | 178 ++
sdks/java/io/hadoop-common/pom.xml | 14 +-
.../beam/sdk/io/hadoop/WritableCoder.java | 107 +-
.../beam/sdk/io/hadoop/WritableCoderTest.java | 10 +
sdks/java/io/hadoop-file-system/pom.xml | 190 ++
.../beam/sdk/io/hdfs/HadoopFileSystem.java | 244 ++
.../sdk/io/hdfs/HadoopFileSystemModule.java | 84 +
.../sdk/io/hdfs/HadoopFileSystemOptions.java | 111 +
.../hdfs/HadoopFileSystemOptionsRegistrar.java | 35 +
.../sdk/io/hdfs/HadoopFileSystemRegistrar.java | 65 +
.../beam/sdk/io/hdfs/HadoopResourceId.java | 95 +
.../apache/beam/sdk/io/hdfs/package-info.java | 23 +
.../sdk/io/hdfs/HadoopFileSystemModuleTest.java | 65 +
.../HadoopFileSystemOptionsRegistrarTest.java | 49 +
.../io/hdfs/HadoopFileSystemOptionsTest.java | 173 ++
.../io/hdfs/HadoopFileSystemRegistrarTest.java | 81 +
.../beam/sdk/io/hdfs/HadoopFileSystemTest.java | 246 ++
.../beam/sdk/io/hdfs/HadoopResourceIdTest.java | 71 +
sdks/java/io/hadoop/README.md | 167 --
sdks/java/io/hadoop/input-format/pom.xml | 17 +-
.../hadoop/inputformat/HadoopInputFormatIO.java | 14 +-
.../ConfigurableEmployeeInputFormat.java | 1 -
.../hadoop/inputformat/EmployeeInputFormat.java | 1 -
.../inputformat/HadoopInputFormatIOTest.java | 46 +-
.../ReuseObjectsEmployeeInputFormat.java | 1 -
.../hadoop/inputformat/TestEmployeeDataSet.java | 2 -
sdks/java/io/hadoop/jdk1.8-tests/pom.xml | 36 +-
.../inputformat/HIFIOWithElasticTest.java | 1 -
.../HIFIOWithEmbeddedCassandraTest.java | 2 -
.../integration/tests/HIFIOCassandraIT.java | 2 -
.../integration/tests/HIFIOElasticIT.java | 1 -
sdks/java/io/hadoop/pom.xml | 23 +-
sdks/java/io/hbase/pom.xml | 34 +-
.../org/apache/beam/sdk/io/hbase/HBaseIO.java | 42 +-
.../beam/sdk/io/hbase/HBaseMutationCoder.java | 7 +-
.../beam/sdk/io/hbase/HBaseResultCoder.java | 5 +-
.../apache/beam/sdk/io/hbase/HBaseIOTest.java | 23 +-
sdks/java/io/hdfs/README.md | 43 -
sdks/java/io/hdfs/pom.xml | 141 -
.../apache/beam/sdk/io/hdfs/HDFSFileSink.java | 478 ---
.../apache/beam/sdk/io/hdfs/HDFSFileSource.java | 625 ----
.../beam/sdk/io/hdfs/HadoopFileSystem.java | 71 -
.../sdk/io/hdfs/HadoopFileSystemRegistrar.java | 42 -
.../beam/sdk/io/hdfs/HadoopResourceId.java | 42 -
.../org/apache/beam/sdk/io/hdfs/UGIHelper.java | 38 -
.../apache/beam/sdk/io/hdfs/package-info.java | 22 -
.../beam/sdk/io/hdfs/HDFSFileSinkTest.java | 173 --
.../beam/sdk/io/hdfs/HDFSFileSourceTest.java | 231 --
.../io/hdfs/HadoopFileSystemRegistrarTest.java | 52 -
sdks/java/io/jdbc/pom.xml | 2 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 18 +-
sdks/java/io/jdbc/src/test/README.md | 32 -
.../org/apache/beam/sdk/io/jdbc/JdbcIOIT.java | 1 -
sdks/java/io/jms/pom.xml | 2 +-
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 23 +-
.../org/apache/beam/sdk/io/jms/JmsRecord.java | 1 -
.../org/apache/beam/sdk/io/jms/JmsIOTest.java | 4 +-
sdks/java/io/kafka/pom.xml | 7 +-
.../beam/sdk/io/kafka/KafkaCheckpointMark.java | 2 -
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 432 ++-
.../apache/beam/sdk/io/kafka/KafkaRecord.java | 1 -
.../beam/sdk/io/kafka/KafkaRecordCoder.java | 33 +-
.../serialization/InstantDeserializer.java | 44 +
.../kafka/serialization/InstantSerializer.java | 44 +
.../io/kafka/serialization/package-info.java | 22 +
.../apache/beam/sdk/io/kafka/KafkaIOTest.java | 297 +-
sdks/java/io/kinesis/pom.xml | 8 +-
.../apache/beam/sdk/io/kinesis/KinesisIO.java | 148 +-
.../beam/sdk/io/kinesis/KinesisRecordCoder.java | 38 +-
.../beam/sdk/io/kinesis/KinesisSource.java | 2 +-
.../sdk/io/kinesis/KinesisMockReadTest.java | 14 +-
.../beam/sdk/io/kinesis/KinesisReaderIT.java | 10 +-
sdks/java/io/mongodb/pom.xml | 2 +-
.../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 11 +-
.../apache/beam/sdk/io/mongodb/MongoDbIO.java | 63 +-
.../sdk/io/mongodb/MongoDBGridFSIOTest.java | 5 +-
.../beam/sdk/io/mongodb/MongoDbIOTest.java | 20 +-
sdks/java/io/mqtt/pom.xml | 2 +-
.../org/apache/beam/sdk/io/mqtt/MqttIO.java | 9 +-
sdks/java/io/pom.xml | 5 +-
sdks/java/io/xml/pom.xml | 113 +
.../org/apache/beam/sdk/io/xml/JAXBCoder.java | 192 ++
.../java/org/apache/beam/sdk/io/xml/XmlIO.java | 544 ++++
.../org/apache/beam/sdk/io/xml/XmlSink.java | 169 ++
.../org/apache/beam/sdk/io/xml/XmlSource.java | 399 +++
.../apache/beam/sdk/io/xml/package-info.java | 22 +
.../apache/beam/sdk/io/xml/JAXBCoderTest.java | 230 ++
.../org/apache/beam/sdk/io/xml/XmlSinkTest.java | 280 ++
.../apache/beam/sdk/io/xml/XmlSourceTest.java | 987 +++++++
sdks/java/java8tests/pom.xml | 21 +-
.../beam/sdk/transforms/DistinctJava8Test.java | 8 -
.../beam/sdk/transforms/FilterJava8Test.java | 2 +-
.../beam/sdk/transforms/PartitionJava8Test.java | 2 +-
.../beam/sdk/transforms/WithKeysJava8Test.java | 9 -
sdks/java/javadoc/README.md | 23 -
sdks/java/javadoc/ant.xml | 5 +-
sdks/java/javadoc/pom.xml | 11 +-
.../examples-java8/generate-sources.sh | 12 +-
.../maven-archetypes/examples-java8/pom.xml | 108 +-
.../main/resources/archetype-resources/pom.xml | 32 +-
.../examples/generate-sources.sh | 8 +-
sdks/java/maven-archetypes/examples/pom.xml | 67 +-
.../main/resources/archetype-resources/pom.xml | 25 +-
sdks/java/maven-archetypes/pom.xml | 57 +-
sdks/java/maven-archetypes/starter/pom.xml | 19 +-
.../main/resources/archetype-resources/pom.xml | 2 +-
.../resources/projects/basic/reference/pom.xml | 2 +-
sdks/java/pom.xml | 2 +-
sdks/pom.xml | 20 +-
sdks/python/.pylintrc | 1 +
sdks/python/MANIFEST.in | 2 +
sdks/python/README.md | 298 --
sdks/python/apache_beam/__init__.py | 2 +-
sdks/python/apache_beam/coders/coder_impl.py | 59 +-
sdks/python/apache_beam/coders/coders.py | 48 +-
sdks/python/apache_beam/coders/coders_test.py | 12 +-
.../apache_beam/coders/coders_test_common.py | 9 +-
.../apache_beam/coders/fast_coders_test.py | 1 +
sdks/python/apache_beam/coders/observable.py | 9 +-
.../apache_beam/coders/slow_coders_test.py | 1 +
sdks/python/apache_beam/coders/slow_stream.py | 21 +-
.../apache_beam/coders/standard_coders_test.py | 4 +-
sdks/python/apache_beam/coders/stream.pyx | 5 +
sdks/python/apache_beam/coders/typecoders.py | 7 +-
sdks/python/apache_beam/error.py | 4 +
.../examples/complete/autocomplete.py | 4 +-
.../examples/complete/autocomplete_test.py | 7 +-
.../examples/complete/estimate_pi.py | 4 +-
.../examples/complete/estimate_pi_test.py | 6 +-
.../examples/complete/game/README.md | 69 -
.../examples/complete/game/hourly_team_score.py | 7 +-
.../complete/game/hourly_team_score_test.py | 7 +-
.../examples/complete/game/user_score.py | 5 +-
.../examples/complete/game/user_score_test.py | 7 +-
.../complete/juliaset/juliaset/juliaset_test.py | 1 +
.../apache_beam/examples/complete/tfidf.py | 4 +-
.../apache_beam/examples/complete/tfidf_test.py | 6 +-
.../examples/complete/top_wikipedia_sessions.py | 4 +-
.../complete/top_wikipedia_sessions_test.py | 6 +-
.../examples/cookbook/bigquery_side_input.py | 6 +-
.../cookbook/bigquery_side_input_test.py | 6 +-
.../cookbook/bigquery_tornadoes_it_test.py | 5 +-
.../cookbook/bigquery_tornadoes_test.py | 8 +-
.../apache_beam/examples/cookbook/bigshuffle.py | 94 -
.../examples/cookbook/bigshuffle_test.py | 63 -
.../apache_beam/examples/cookbook/coders.py | 4 +-
.../examples/cookbook/coders_test.py | 6 +-
.../examples/cookbook/combiners_test.py | 8 +-
.../examples/cookbook/custom_ptransform.py | 2 +-
.../examples/cookbook/custom_ptransform_test.py | 6 +-
.../examples/cookbook/datastore_wordcount.py | 34 +-
.../apache_beam/examples/cookbook/filters.py | 2 +-
.../examples/cookbook/filters_test.py | 14 +-
.../examples/cookbook/group_with_coder.py | 4 +-
.../examples/cookbook/mergecontacts.py | 18 +-
.../examples/cookbook/multiple_output_pardo.py | 51 +-
.../apache_beam/examples/snippets/snippets.py | 173 +-
.../examples/snippets/snippets_test.py | 110 +-
sdks/python/apache_beam/examples/wordcount.py | 13 +-
.../apache_beam/examples/wordcount_debugging.py | 10 +-
.../examples/wordcount_debugging_test.py | 1 +
.../apache_beam/examples/wordcount_it_test.py | 6 +-
.../apache_beam/examples/wordcount_minimal.py | 4 +-
sdks/python/apache_beam/internal/__init__.py | 2 +
.../python/apache_beam/internal/gcp/__init__.py | 2 +
sdks/python/apache_beam/internal/gcp/auth.py | 50 +-
.../apache_beam/internal/gcp/json_value.py | 18 +-
.../apache_beam/internal/gcp/json_value_test.py | 11 +
sdks/python/apache_beam/internal/module_test.py | 1 +
sdks/python/apache_beam/internal/pickler.py | 31 +-
.../python/apache_beam/internal/pickler_test.py | 1 +
sdks/python/apache_beam/internal/util.py | 21 +-
sdks/python/apache_beam/io/__init__.py | 2 +-
sdks/python/apache_beam/io/avroio.py | 17 +-
sdks/python/apache_beam/io/avroio_test.py | 6 +-
sdks/python/apache_beam/io/concat_source.py | 86 +-
.../python/apache_beam/io/concat_source_test.py | 9 +-
sdks/python/apache_beam/io/filebasedsink.py | 299 ++
.../python/apache_beam/io/filebasedsink_test.py | 303 ++
sdks/python/apache_beam/io/filebasedsource.py | 49 +-
.../apache_beam/io/filebasedsource_test.py | 48 +-
sdks/python/apache_beam/io/fileio.py | 358 ---
sdks/python/apache_beam/io/fileio_test.py | 231 --
sdks/python/apache_beam/io/filesystem.py | 54 +-
sdks/python/apache_beam/io/filesystems.py | 236 ++
sdks/python/apache_beam/io/filesystems_test.py | 237 ++
sdks/python/apache_beam/io/filesystems_util.py | 35 -
sdks/python/apache_beam/io/gcp/bigquery.py | 80 +-
sdks/python/apache_beam/io/gcp/bigquery_test.py | 2 +-
.../io/gcp/datastore/v1/datastoreio.py | 10 +-
.../io/gcp/datastore/v1/datastoreio_test.py | 1 +
.../io/gcp/datastore/v1/fake_datastore.py | 14 +-
.../apache_beam/io/gcp/datastore/v1/helper.py | 35 +-
.../io/gcp/datastore/v1/helper_test.py | 26 +-
.../io/gcp/datastore/v1/query_splitter_test.py | 37 +-
sdks/python/apache_beam/io/gcp/gcsfilesystem.py | 56 +-
.../apache_beam/io/gcp/gcsfilesystem_test.py | 34 +
sdks/python/apache_beam/io/gcp/gcsio.py | 9 +-
sdks/python/apache_beam/io/gcp/pubsub.py | 2 +
sdks/python/apache_beam/io/gcp/pubsub_test.py | 1 +
.../io/gcp/tests/bigquery_matcher.py | 5 +-
.../io/gcp/tests/bigquery_matcher_test.py | 2 +-
sdks/python/apache_beam/io/iobase.py | 28 +-
sdks/python/apache_beam/io/localfilesystem.py | 34 +-
.../apache_beam/io/localfilesystem_test.py | 72 +-
sdks/python/apache_beam/io/range_trackers.py | 18 +-
sdks/python/apache_beam/io/source_test_utils.py | 16 +-
.../apache_beam/io/source_test_utils_test.py | 1 +
sdks/python/apache_beam/io/sources_test.py | 8 +-
sdks/python/apache_beam/io/textio.py | 32 +-
sdks/python/apache_beam/io/textio_test.py | 7 +-
sdks/python/apache_beam/io/tfrecordio.py | 21 +-
sdks/python/apache_beam/io/tfrecordio_test.py | 35 +-
sdks/python/apache_beam/metrics/__init__.py | 1 +
sdks/python/apache_beam/metrics/cells.py | 50 +-
sdks/python/apache_beam/metrics/execution.py | 7 +-
sdks/python/apache_beam/metrics/metric.py | 4 +
sdks/python/apache_beam/metrics/metricbase.py | 2 +
sdks/python/apache_beam/options/__init__.py | 16 +
.../apache_beam/options/pipeline_options.py | 650 +++++
.../options/pipeline_options_test.py | 240 ++
.../options/pipeline_options_validator.py | 201 ++
.../options/pipeline_options_validator_test.py | 343 +++
.../apache_beam/options/value_provider.py | 111 +
.../apache_beam/options/value_provider_test.py | 145 +
sdks/python/apache_beam/pipeline.py | 85 +-
sdks/python/apache_beam/pipeline_test.py | 22 +-
sdks/python/apache_beam/pvalue.py | 28 +-
sdks/python/apache_beam/pvalue_test.py | 2 +-
sdks/python/apache_beam/runners/api/__init__.py | 16 +
.../apache_beam/runners/api/beam_fn_api_pb2.py | 2742 ++++++++++++++++++
.../runners/api/beam_fn_api_pb2_grpc.py | 205 ++
.../runners/api/beam_runner_api_pb2.py | 300 +-
sdks/python/apache_beam/runners/common.pxd | 64 +-
sdks/python/apache_beam/runners/common.py | 538 ++--
sdks/python/apache_beam/runners/common_test.py | 58 +
.../apache_beam/runners/dataflow/__init__.py | 9 +
.../runners/dataflow/dataflow_metrics.py | 113 +-
.../runners/dataflow/dataflow_metrics_test.py | 198 +-
.../runners/dataflow/dataflow_runner.py | 107 +-
.../runners/dataflow/dataflow_runner_test.py | 72 +-
.../runners/dataflow/internal/apiclient.py | 33 +-
.../runners/dataflow/internal/apiclient_test.py | 23 +-
.../clients/dataflow/dataflow_v1b3_messages.py | 564 +++-
.../runners/dataflow/internal/dependency.py | 51 +-
.../dataflow/internal/dependency_test.py | 12 +-
.../runners/dataflow/internal/names.py | 8 +-
.../runners/dataflow/native_io/iobase.py | 3 +
.../runners/dataflow/template_runner_test.py | 4 +-
.../runners/dataflow/test_dataflow_runner.py | 9 +-
.../apache_beam/runners/direct/__init__.py | 6 +-
.../runners/direct/bundle_factory.py | 16 +-
sdks/python/apache_beam/runners/direct/clock.py | 14 +-
.../consumer_tracking_pipeline_visitor.py | 4 +-
.../consumer_tracking_pipeline_visitor_test.py | 2 +-
.../apache_beam/runners/direct/direct_runner.py | 15 +-
.../runners/direct/evaluation_context.py | 11 +-
.../apache_beam/runners/direct/executor.py | 79 +-
.../runners/direct/transform_evaluator.py | 32 +-
.../runners/direct/transform_result.py | 49 +-
.../runners/direct/watermark_manager.py | 18 +-
.../apache_beam/runners/pipeline_context.py | 10 +-
.../apache_beam/runners/portability/__init__.py | 18 +
.../runners/portability/fn_api_runner.py | 473 +++
.../runners/portability/fn_api_runner_test.py | 40 +
.../portability/maptask_executor_runner.py | 471 +++
.../portability/maptask_executor_runner_test.py | 225 ++
sdks/python/apache_beam/runners/runner.py | 90 +-
sdks/python/apache_beam/runners/runner_test.py | 47 +-
.../apache_beam/runners/worker/__init__.py | 18 +
.../apache_beam/runners/worker/data_plane.py | 290 ++
.../runners/worker/data_plane_test.py | 139 +
.../apache_beam/runners/worker/log_handler.py | 102 +
.../runners/worker/log_handler_test.py | 105 +
.../apache_beam/runners/worker/logger.pxd | 25 +
.../python/apache_beam/runners/worker/logger.py | 175 ++
.../apache_beam/runners/worker/logger_test.py | 182 ++
.../apache_beam/runners/worker/opcounters.pxd | 45 +
.../apache_beam/runners/worker/opcounters.py | 164 ++
.../runners/worker/opcounters_test.py | 149 +
.../runners/worker/operation_specs.py | 370 +++
.../apache_beam/runners/worker/operations.pxd | 89 +
.../apache_beam/runners/worker/operations.py | 651 +++++
.../apache_beam/runners/worker/sdk_worker.py | 455 +++
.../runners/worker/sdk_worker_main.py | 64 +
.../runners/worker/sdk_worker_test.py | 168 ++
.../apache_beam/runners/worker/sideinputs.py | 168 ++
.../runners/worker/sideinputs_test.py | 150 +
.../apache_beam/runners/worker/statesampler.pyx | 237 ++
.../runners/worker/statesampler_fake.py | 36 +
.../runners/worker/statesampler_test.py | 102 +
sdks/python/apache_beam/test_pipeline.py | 163 --
sdks/python/apache_beam/test_pipeline_test.py | 112 -
sdks/python/apache_beam/testing/__init__.py | 16 +
.../testing/data/standard_coders.yaml | 196 ++
.../testing/data/trigger_transcripts.yaml | 224 ++
.../apache_beam/testing/pipeline_verifiers.py | 154 +
.../testing/pipeline_verifiers_test.py | 148 +
.../python/apache_beam/testing/test_pipeline.py | 168 ++
.../apache_beam/testing/test_pipeline_test.py | 112 +
sdks/python/apache_beam/testing/test_stream.py | 175 ++
.../apache_beam/testing/test_stream_test.py | 83 +
sdks/python/apache_beam/testing/test_utils.py | 73 +
sdks/python/apache_beam/testing/util.py | 107 +
sdks/python/apache_beam/testing/util_test.py | 50 +
sdks/python/apache_beam/tests/__init__.py | 16 -
sdks/python/apache_beam/tests/data/README.md | 20 -
.../apache_beam/tests/data/privatekey.p12 | Bin 2452 -> 0 bytes
.../apache_beam/tests/data/standard_coders.yaml | 196 --
.../apache_beam/tests/pipeline_verifiers.py | 147 -
.../tests/pipeline_verifiers_test.py | 148 -
sdks/python/apache_beam/tests/test_utils.py | 69 -
sdks/python/apache_beam/transforms/__init__.py | 2 +-
.../apache_beam/transforms/combiners_test.py | 4 +-
sdks/python/apache_beam/transforms/core.py | 182 +-
.../apache_beam/transforms/create_test.py | 5 +-
.../apache_beam/transforms/cy_combiners.py | 5 +-
sdks/python/apache_beam/transforms/display.py | 3 +-
.../apache_beam/transforms/display_test.py | 38 +-
.../python/apache_beam/transforms/ptransform.py | 87 +-
.../apache_beam/transforms/ptransform_test.py | 221 +-
.../python/apache_beam/transforms/sideinputs.py | 2 +
.../apache_beam/transforms/sideinputs_test.py | 4 +-
sdks/python/apache_beam/transforms/timeutil.py | 33 +-
sdks/python/apache_beam/transforms/trigger.py | 126 +-
.../apache_beam/transforms/trigger_test.py | 60 +-
.../transforms/trigger_transcripts.yaml | 224 --
sdks/python/apache_beam/transforms/util.py | 79 -
sdks/python/apache_beam/transforms/util_test.py | 50 -
sdks/python/apache_beam/transforms/window.py | 165 +-
.../apache_beam/transforms/window_test.py | 28 +-
.../transforms/write_ptransform_test.py | 4 +-
sdks/python/apache_beam/typehints/decorators.py | 18 +-
sdks/python/apache_beam/typehints/opcodes.py | 7 +-
.../apache_beam/typehints/trivial_inference.py | 2 +
sdks/python/apache_beam/typehints/typecheck.py | 21 +-
.../typehints/typed_pipeline_test.py | 6 +-
sdks/python/apache_beam/typehints/typehints.py | 18 +
.../apache_beam/typehints/typehints_test.py | 20 +-
sdks/python/apache_beam/utils/__init__.py | 7 +-
sdks/python/apache_beam/utils/annotations.py | 18 +-
.../apache_beam/utils/annotations_test.py | 29 +
sdks/python/apache_beam/utils/counters.py | 5 +-
sdks/python/apache_beam/utils/path.py | 46 -
sdks/python/apache_beam/utils/path_test.py | 70 -
.../apache_beam/utils/pipeline_options.py | 559 ----
.../apache_beam/utils/pipeline_options_test.py | 192 --
.../utils/pipeline_options_validator.py | 199 --
.../utils/pipeline_options_validator_test.py | 342 ---
sdks/python/apache_beam/utils/processes.py | 6 +-
sdks/python/apache_beam/utils/profiler.py | 5 +-
sdks/python/apache_beam/utils/proto_utils.py | 4 +-
sdks/python/apache_beam/utils/retry.py | 2 +
sdks/python/apache_beam/utils/test_stream.py | 163 --
.../apache_beam/utils/test_stream_test.py | 82 -
sdks/python/apache_beam/utils/timestamp.py | 9 +-
sdks/python/apache_beam/utils/urns.py | 95 +
sdks/python/apache_beam/utils/windowed_value.py | 4 +-
.../apache_beam/utils/windowed_value_test.py | 2 +-
sdks/python/apache_beam/version.py | 2 +-
sdks/python/generate_pydoc.sh | 13 +-
sdks/python/pom.xml | 4 +-
sdks/python/run_postcommit.sh | 4 +-
sdks/python/run_pylint.sh | 32 +-
sdks/python/setup.py | 13 +-
sdks/python/tox.ini | 11 +-
1744 files changed, 83941 insertions(+), 77236 deletions(-)
----------------------------------------------------------------------