You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2017/08/07 23:53:40 UTC
[15/50] [abbrv] beam git commit: Merge branch 'master' of
https://github.com/apache/incubator-beam into gearpump-runner
Merge branch 'master' of https://github.com/apache/incubator-beam into gearpump-runner
# Conflicts:
# .test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
# runners/pom.xml
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/46c41fce
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/46c41fce
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/46c41fce
Branch: refs/heads/master
Commit: 46c41fce0e130bdd718c0d3ce7c31a564524bae6
Parents: ebbb613 104f982
Author: manuzhang <ow...@gmail.com>
Authored: Thu Apr 20 20:59:22 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Thu Apr 20 20:59:22 2017 +0800
----------------------------------------------------------------------
.github/PULL_REQUEST_TEMPLATE.md | 2 +-
.gitignore | 3 +
.jenkins/common_job_properties.groovy | 223 --
...job_beam_PostCommit_Java_MavenInstall.groovy | 42 -
...ostCommit_Java_RunnableOnService_Apex.groovy | 47 -
...ommit_Java_RunnableOnService_Dataflow.groovy | 45 -
...stCommit_Java_RunnableOnService_Flink.groovy | 44 -
...ommit_Java_RunnableOnService_Gearpump.groovy | 49 -
...stCommit_Java_RunnableOnService_Spark.groovy | 44 -
.../job_beam_PostCommit_Python_Verify.groovy | 43 -
.../job_beam_PreCommit_Java_MavenInstall.groovy | 42 -
.../job_beam_PreCommit_Website_Stage.groovy | 80 -
.jenkins/job_beam_PreCommit_Website_Test.groovy | 65 -
.../job_beam_Release_NightlySnapshot.groovy | 45 -
.jenkins/job_seed.groovy | 53 -
.../jenkins/common_job_properties.groovy | 261 ++
.../job_beam_PerformanceTests_Dataflow.groovy | 43 +
.../job_beam_PerformanceTests_JDBC.groovy | 60 +
.../job_beam_PerformanceTests_Spark.groovy | 44 +
...job_beam_PostCommit_Java_MavenInstall.groovy | 42 +
..._PostCommit_Java_ValidatesRunner_Apex.groovy | 48 +
...tCommit_Java_ValidatesRunner_Dataflow.groovy | 45 +
...PostCommit_Java_ValidatesRunner_Flink.groovy | 43 +
...tCommit_Java_ValidatesRunner_Gearpump.groovy | 49 +
...PostCommit_Java_ValidatesRunner_Spark.groovy | 44 +
.../job_beam_PostCommit_Python_Verify.groovy | 55 +
.../job_beam_PreCommit_Java_MavenInstall.groovy | 42 +
.../job_beam_PreCommit_Website_Stage.groovy | 80 +
.../job_beam_PreCommit_Website_Test.groovy | 65 +
.../job_beam_Release_NightlySnapshot.groovy | 45 +
.test-infra/jenkins/job_seed.groovy | 53 +
.../cassandra-service-for-local-dev.yaml | 28 +
.../cassandra-svc-statefulset.yaml | 114 +
.../LargeITCluster/cassandra-svc-temp.yaml | 74 +
.../cassandra/LargeITCluster/data-load.sh | 122 +
.../cassandra/LargeITCluster/show_health.sh | 47 +
.../cassandra/LargeITCluster/start-up.sh | 22 +
.../cassandra/LargeITCluster/teardown.sh | 25 +
.../cassandra-service-for-local-dev.yaml | 30 +
.../SmallITCluster/cassandra-svc-rc.yaml | 74 +
.../cassandra/SmallITCluster/data-load.sh | 86 +
.../cassandra/SmallITCluster/show_health.sh | 47 +
.../cassandra/SmallITCluster/start-up.sh | 23 +
.../cassandra/SmallITCluster/teardown.sh | 22 +
.../kubernetes/cassandra/data-load-setup.sh | 29 +
.../elasticsearch-service-for-local-dev.yaml | 33 +
.../es-services-deployments.yaml | 258 ++
.../LargeProductionCluster/start-up.sh | 22 +
.../LargeProductionCluster/teardown.sh | 21 +
.../elasticsearch-service-for-local-dev.yaml | 34 +
.../SmallITCluster/elasticsearch-svc-rc.yaml | 96 +
.../elasticsearch/SmallITCluster/start-up.sh | 23 +
.../elasticsearch/SmallITCluster/teardown.sh | 21 +
.../kubernetes/elasticsearch/data-load-setup.sh | 26 +
.../kubernetes/elasticsearch/data-load.sh | 33 +
.../kubernetes/elasticsearch/es_test_data.py | 299 ++
.../kubernetes/elasticsearch/show-health.sh | 33 +
.../postgres/postgres-service-for-local-dev.yml | 28 +
.test-infra/kubernetes/postgres/postgres.yml | 56 +
.test-infra/travis/README.md | 23 +
.test-infra/travis/settings.xml | 33 +
.test-infra/travis/test_wordcount.sh | 125 +
.travis.yml | 7 +-
.travis/README.md | 23 -
.travis/settings.xml | 33 -
.travis/test_wordcount.sh | 125 -
README.md | 3 +
examples/java/README.md | 61 +-
examples/java/pom.xml | 32 +-
.../beam/examples/DebuggingWordCount.java | 2 +-
.../apache/beam/examples/WindowedWordCount.java | 34 +-
.../examples/common/WriteOneFilePerWindow.java | 91 +
.../examples/common/WriteWindowedFilesDoFn.java | 77 -
.../beam/examples/complete/AutoComplete.java | 2 +-
.../examples/complete/StreamingWordExtract.java | 2 +-
.../apache/beam/examples/complete/TfIdf.java | 3 +-
.../examples/complete/TrafficMaxLaneFlow.java | 2 +-
.../beam/examples/complete/TrafficRoutes.java | 2 +-
.../examples/cookbook/BigQueryTornadoes.java | 12 +-
.../cookbook/CombinePerKeyExamples.java | 4 +-
.../beam/examples/cookbook/FilterExamples.java | 7 +-
.../beam/examples/cookbook/JoinExamples.java | 4 +-
.../examples/cookbook/MaxPerKeyExamples.java | 4 +-
.../org/apache/beam/examples/cookbook/README.md | 2 +-
.../beam/examples/cookbook/TriggerExample.java | 4 +-
.../beam/examples/WindowedWordCountIT.java | 41 +-
.../org/apache/beam/examples/WordCountTest.java | 4 +-
.../beam/examples/complete/TfIdfTest.java | 4 +-
.../complete/TopWikipediaSessionsTest.java | 4 +-
.../examples/cookbook/DistinctExampleTest.java | 6 +-
.../examples/cookbook/JoinExamplesTest.java | 4 +-
.../examples/cookbook/TriggerExampleTest.java | 4 +-
examples/java8/pom.xml | 41 +-
.../beam/examples/MinimalWordCountJava8.java | 9 +-
.../beam/examples/complete/game/GameStats.java | 16 +-
.../examples/complete/game/LeaderBoard.java | 2 +-
.../beam/examples/complete/game/README.md | 6 +-
.../beam/examples/complete/game/UserScore.java | 5 +-
.../complete/game/injector/Injector.java | 3 +-
.../complete/game/utils/WriteToBigQuery.java | 13 +-
.../game/utils/WriteWindowedToBigQuery.java | 9 +-
.../examples/MinimalWordCountJava8Test.java | 9 +-
.../examples/complete/game/GameStatsTest.java | 6 +-
.../complete/game/HourlyTeamScoreTest.java | 11 +-
.../examples/complete/game/UserScoreTest.java | 14 +-
pom.xml | 161 +-
runners/apex/pom.xml | 12 +-
.../apache/beam/runners/apex/ApexRunner.java | 81 +-
.../beam/runners/apex/ApexYarnLauncher.java | 111 +-
.../translation/ApexPipelineTranslator.java | 5 +-
.../FlattenPCollectionTranslator.java | 13 +-
.../apex/translation/GroupByKeyTranslator.java | 4 +-
.../translation/ParDoBoundMultiTranslator.java | 185 -
.../apex/translation/ParDoBoundTranslator.java | 95 -
.../apex/translation/ParDoTranslator.java | 185 +
.../apex/translation/TranslationContext.java | 22 +-
.../translation/WindowAssignTranslator.java | 58 +-
.../operators/ApexGroupByKeyOperator.java | 273 +-
.../operators/ApexParDoOperator.java | 238 +-
.../operators/ApexProcessFnOperator.java | 184 +
.../translation/utils/ApexStateInternals.java | 101 +-
.../apex/translation/utils/NoOpStepContext.java | 2 +-
.../utils/SerializablePipelineOptions.java | 13 +-
.../translation/utils/StateInternalsProxy.java | 67 +
.../apex/translation/utils/ValuesSource.java | 2 +-
.../apex/examples/UnboundedTextSource.java | 2 +-
.../translation/ApexGroupByKeyOperatorTest.java | 2 +-
.../FlattenPCollectionTranslatorTest.java | 3 +-
.../translation/GroupByKeyTranslatorTest.java | 2 +-
.../translation/ParDoBoundTranslatorTest.java | 344 --
.../apex/translation/ParDoTranslatorTest.java | 345 ++
.../utils/ApexStateInternalsTest.java | 43 +-
.../translation/utils/CollectionSource.java | 2 +-
runners/core-construction-java/pom.xml | 97 +-
.../beam/runners/core/construction/Coders.java | 174 +
.../DeduplicatedFlattenFactory.java | 120 +
.../EmptyFlattenAsCreateFactory.java | 34 +-
.../core/construction/ForwardingPTransform.java | 62 +
.../runners/core/construction/PCollections.java | 97 +
.../core/construction/PTransformMatchers.java | 135 +-
.../construction/PTransformReplacements.java | 69 +
.../runners/core/construction/PTransforms.java | 107 +
.../core/construction/PrimitiveCreate.java | 18 +-
.../core/construction/ReplacementOutputs.java | 63 +-
.../core/construction/SdkComponents.java | 195 ++
.../SingleInputOutputOverrideFactory.java | 14 +-
.../runners/core/construction/Triggers.java | 336 ++
.../UnboundedReadFromBoundedSource.java | 543 +++
.../core/construction/UnconsumedReads.java | 72 +
.../UnsupportedOverrideFactory.java | 20 +-
.../core/construction/WindowingStrategies.java | 245 ++
.../runners/core/construction/CodersTest.java | 163 +
.../DeduplicatedFlattenFactoryTest.java | 104 +
.../EmptyFlattenAsCreateFactoryTest.java | 122 +
.../construction/ForwardingPTransformTest.java | 111 +
.../core/construction/PCollectionsTest.java | 188 +
.../construction/PTransformMatchersTest.java | 198 +-
.../PTransformReplacementsTest.java | 131 +
.../core/construction/PTransformsTest.java | 189 +
.../construction/ReplacementOutputsTest.java | 109 +-
.../core/construction/SdkComponentsTest.java | 223 ++
.../SingleInputOutputOverrideFactoryTest.java | 37 +-
.../runners/core/construction/TriggersTest.java | 111 +
.../UnboundedReadFromBoundedSourceTest.java | 373 ++
.../core/construction/UnconsumedReadsTest.java | 105 +
.../UnsupportedOverrideFactoryTest.java | 16 +-
.../construction/WindowingStrategiesTest.java | 110 +
runners/core-java/pom.xml | 58 +-
.../beam/runners/core/BaseExecutionContext.java | 13 +-
.../apache/beam/runners/core/DoFnAdapters.java | 21 +-
.../apache/beam/runners/core/DoFnRunners.java | 40 +-
.../beam/runners/core/ExecutionContext.java | 13 +-
.../GroupAlsoByWindowViaOutputBufferDoFn.java | 2 +-
.../core/GroupAlsoByWindowViaWindowSetDoFn.java | 2 +-
.../GroupAlsoByWindowViaWindowSetNewDoFn.java | 8 +-
.../runners/core/InMemoryStateInternals.java | 120 +-
.../apache/beam/runners/core/NonEmptyPanes.java | 4 +-
.../org/apache/beam/runners/core/OldDoFn.java | 49 +-
...eBoundedSplittableProcessElementInvoker.java | 136 +-
.../beam/runners/core/OutputWindowedValue.java | 10 +-
.../beam/runners/core/ProcessFnRunner.java | 127 +
.../core/PushbackSideInputDoFnRunner.java | 106 +-
.../runners/core/ReduceFnContextFactory.java | 3 +-
.../beam/runners/core/SideInputHandler.java | 18 +-
.../beam/runners/core/SimpleDoFnRunner.java | 76 +-
.../beam/runners/core/SimpleOldDoFnRunner.java | 65 +-
.../core/SimplePushbackSideInputDoFnRunner.java | 115 +
.../beam/runners/core/SplittableParDo.java | 154 +-
.../core/SplittableProcessElementInvoker.java | 22 +-
.../apache/beam/runners/core/StateMerging.java | 16 +-
.../org/apache/beam/runners/core/StateTag.java | 14 +-
.../org/apache/beam/runners/core/StateTags.java | 32 +-
.../beam/runners/core/StatefulDoFnRunner.java | 96 +
.../beam/runners/core/SystemReduceFn.java | 8 +-
.../core/UnboundedReadFromBoundedSource.java | 536 ---
.../beam/runners/core/WindowingInternals.java | 8 +-
.../core/WindowingInternalsAdapters.java | 8 +-
.../AfterDelayFromFirstElementStateMachine.java | 10 +-
.../core/triggers/AfterPaneStateMachine.java | 4 +-
.../triggers/AfterWatermarkStateMachine.java | 14 +-
.../core/GroupAlsoByWindowsProperties.java | 10 +-
.../core/InMemoryStateInternalsTest.java | 105 +-
.../apache/beam/runners/core/NoOpOldDoFn.java | 4 +-
.../apache/beam/runners/core/OldDoFnTest.java | 4 +-
...ndedSplittableProcessElementInvokerTest.java | 27 +-
.../core/PushbackSideInputDoFnRunnerTest.java | 282 --
.../beam/runners/core/ReduceFnRunnerTest.java | 10 +-
.../beam/runners/core/ReduceFnTester.java | 10 +-
.../beam/runners/core/SideInputHandlerTest.java | 22 +-
.../beam/runners/core/SimpleDoFnRunnerTest.java | 145 +
.../runners/core/SimpleOldDoFnRunnerTest.java | 4 +-
.../SimplePushbackSideInputDoFnRunnerTest.java | 282 ++
.../beam/runners/core/SplittableParDoTest.java | 332 +-
.../runners/core/StatefulDoFnRunnerTest.java | 113 +-
.../UnboundedReadFromBoundedSourceTest.java | 373 --
runners/direct-java/pom.xml | 55 +-
.../direct/BoundedReadEvaluatorFactory.java | 4 +-
.../CopyOnAccessInMemoryStateInternals.java | 46 +-
...ectGBKIntoKeyedWorkItemsOverrideFactory.java | 16 +-
.../beam/runners/direct/DirectGraphVisitor.java | 5 +-
.../beam/runners/direct/DirectGroupByKey.java | 1 +
.../direct/DirectGroupByKeyOverrideFactory.java | 14 +-
.../beam/runners/direct/DirectMetrics.java | 129 +-
.../beam/runners/direct/DirectOptions.java | 11 -
.../beam/runners/direct/DirectRunner.java | 132 +-
...ecycleManagerRemovingTransformEvaluator.java | 6 +-
.../beam/runners/direct/EvaluationContext.java | 34 +-
.../direct/ExecutorServiceParallelExecutor.java | 141 +-
.../runners/direct/FlattenEvaluatorFactory.java | 2 +-
.../runners/direct/ForwardingPTransform.java | 62 -
.../GroupAlsoByWindowEvaluatorFactory.java | 12 +-
.../direct/GroupByKeyOnlyEvaluatorFactory.java | 4 +-
.../direct/KeyedPValueTrackingVisitor.java | 18 +-
.../beam/runners/direct/ModelEnforcement.java | 13 +-
.../beam/runners/direct/ParDoEvaluator.java | 129 +-
.../runners/direct/ParDoEvaluatorFactory.java | 39 +-
.../direct/ParDoMultiOverrideFactory.java | 53 +-
.../ParDoSingleViaMultiOverrideFactory.java | 70 -
.../beam/runners/direct/PipelineExecutor.java | 23 +-
...littableProcessElementsEvaluatorFactory.java | 108 +-
.../direct/StatefulParDoEvaluatorFactory.java | 31 +-
.../direct/TestStreamEvaluatorFactory.java | 20 +-
.../direct/TransformEvaluatorFactory.java | 10 +-
.../direct/TransformEvaluatorRegistry.java | 7 +-
.../direct/TransformExecutorService.java | 6 +
.../direct/TransformExecutorServices.java | 57 +-
.../direct/UnboundedReadEvaluatorFactory.java | 39 +-
.../runners/direct/ViewEvaluatorFactory.java | 82 +-
.../runners/direct/ViewOverrideFactory.java | 114 +
.../beam/runners/direct/WatermarkManager.java | 19 +-
.../runners/direct/WindowEvaluatorFactory.java | 2 +-
.../direct/WriteWithShardingFactory.java | 26 +-
.../direct/BoundedReadEvaluatorFactoryTest.java | 4 +-
.../CopyOnAccessInMemoryStateInternalsTest.java | 34 +-
.../runners/direct/DirectGraphVisitorTest.java | 7 +-
.../DirectGroupByKeyOverrideFactoryTest.java | 12 +-
.../beam/runners/direct/DirectMetricsTest.java | 124 +-
.../beam/runners/direct/DirectRunnerTest.java | 89 +-
...leManagerRemovingTransformEvaluatorTest.java | 8 +-
.../runners/direct/EvaluationContextTest.java | 57 +-
.../direct/ForwardingPTransformTest.java | 112 -
.../beam/runners/direct/ParDoEvaluatorTest.java | 15 +-
.../direct/ParDoMultiOverrideFactoryTest.java | 45 -
.../ParDoSingleViaMultiOverrideFactoryTest.java | 46 -
.../runners/direct/SideInputContainerTest.java | 19 +-
.../StatefulParDoEvaluatorFactoryTest.java | 5 +-
.../direct/TestStreamEvaluatorFactoryTest.java | 11 -
.../direct/TransformExecutorServicesTest.java | 48 +
.../UnboundedReadEvaluatorFactoryTest.java | 88 +-
.../direct/ViewEvaluatorFactoryTest.java | 14 +-
.../runners/direct/ViewOverrideFactoryTest.java | 138 +
.../direct/WindowEvaluatorFactoryTest.java | 12 +-
.../direct/WriteWithShardingFactoryTest.java | 28 +-
runners/flink/examples/pom.xml | 126 -
.../beam/runners/flink/examples/TFIDF.java | 456 ---
.../beam/runners/flink/examples/WordCount.java | 129 -
.../runners/flink/examples/package-info.java | 22 -
.../flink/examples/streaming/AutoComplete.java | 400 ---
.../flink/examples/streaming/JoinExamples.java | 154 -
.../examples/streaming/KafkaIOExamples.java | 338 --
.../KafkaWindowedWordCountExample.java | 164 -
.../examples/streaming/WindowedWordCount.java | 141 -
.../flink/examples/streaming/package-info.java | 22 -
runners/flink/pom.xml | 320 +-
runners/flink/runner/pom.xml | 310 --
.../flink/DefaultParallelismFactory.java | 39 -
.../flink/FlinkBatchPipelineTranslator.java | 139 -
.../flink/FlinkBatchTransformTranslators.java | 797 -----
.../flink/FlinkBatchTranslationContext.java | 154 -
.../flink/FlinkDetachedRunnerResult.java | 76 -
.../FlinkPipelineExecutionEnvironment.java | 241 --
.../runners/flink/FlinkPipelineOptions.java | 101 -
.../runners/flink/FlinkPipelineTranslator.java | 53 -
.../apache/beam/runners/flink/FlinkRunner.java | 250 --
.../runners/flink/FlinkRunnerRegistrar.java | 62 -
.../beam/runners/flink/FlinkRunnerResult.java | 98 -
.../flink/FlinkStreamingPipelineTranslator.java | 230 --
.../FlinkStreamingTransformTranslators.java | 1043 ------
.../flink/FlinkStreamingTranslationContext.java | 130 -
.../flink/FlinkStreamingViewOverrides.java | 372 --
.../flink/PipelineTranslationOptimizer.java | 72 -
.../beam/runners/flink/TestFlinkRunner.java | 97 -
.../beam/runners/flink/TranslationMode.java | 31 -
.../apache/beam/runners/flink/package-info.java | 22 -
.../functions/FlinkAggregatorFactory.java | 53 -
.../functions/FlinkAssignContext.java | 63 -
.../functions/FlinkAssignWindows.java | 49 -
.../functions/FlinkDoFnFunction.java | 161 -
.../FlinkMergingNonShuffleReduceFunction.java | 228 --
.../FlinkMergingPartialReduceFunction.java | 201 --
.../functions/FlinkMergingReduceFunction.java | 199 --
.../FlinkMultiOutputPruningFunction.java | 50 -
.../functions/FlinkNoOpStepContext.java | 73 -
.../functions/FlinkPartialReduceFunction.java | 172 -
.../functions/FlinkReduceFunction.java | 173 -
.../functions/FlinkSideInputReader.java | 80 -
.../functions/FlinkStatefulDoFnFunction.java | 198 --
.../functions/SideInputInitializer.java | 73 -
.../translation/functions/package-info.java | 22 -
.../runners/flink/translation/package-info.java | 22 -
.../translation/types/CoderTypeInformation.java | 120 -
.../translation/types/CoderTypeSerializer.java | 132 -
.../types/EncodedValueComparator.java | 195 --
.../types/EncodedValueSerializer.java | 113 -
.../types/EncodedValueTypeInformation.java | 107 -
.../flink/translation/types/FlinkCoder.java | 63 -
.../types/InspectableByteArrayOutputStream.java | 34 -
.../flink/translation/types/KvKeySelector.java | 50 -
.../flink/translation/types/package-info.java | 22 -
.../utils/SerializedPipelineOptions.java | 65 -
.../flink/translation/utils/package-info.java | 22 -
.../wrappers/DataInputViewWrapper.java | 58 -
.../wrappers/DataOutputViewWrapper.java | 51 -
.../SerializableFnAggregatorWrapper.java | 98 -
.../translation/wrappers/SourceInputFormat.java | 149 -
.../translation/wrappers/SourceInputSplit.java | 52 -
.../translation/wrappers/package-info.java | 22 -
.../wrappers/streaming/DoFnOperator.java | 861 -----
.../streaming/KvToByteBufferKeySelector.java | 56 -
.../streaming/SingletonKeyedWorkItem.java | 58 -
.../streaming/SingletonKeyedWorkItemCoder.java | 128 -
.../wrappers/streaming/WindowDoFnOperator.java | 120 -
.../wrappers/streaming/WorkItemKeySelector.java | 56 -
.../streaming/io/BoundedSourceWrapper.java | 218 --
.../streaming/io/UnboundedFlinkSink.java | 182 -
.../streaming/io/UnboundedFlinkSource.java | 120 -
.../streaming/io/UnboundedSocketSource.java | 249 --
.../streaming/io/UnboundedSourceWrapper.java | 476 ---
.../wrappers/streaming/io/package-info.java | 22 -
.../wrappers/streaming/package-info.java | 22 -
.../state/FlinkBroadcastStateInternals.java | 865 -----
.../state/FlinkKeyGroupStateInternals.java | 487 ---
.../state/FlinkSplitStateInternals.java | 260 --
.../streaming/state/FlinkStateInternals.java | 1053 ------
.../state/KeyGroupCheckpointedOperator.java | 35 -
.../state/KeyGroupRestoringOperator.java | 32 -
.../wrappers/streaming/state/package-info.java | 22 -
.../runner/src/main/resources/log4j.properties | 23 -
.../flink/EncodedValueComparatorTest.java | 70 -
.../runners/flink/FlinkRunnerRegistrarTest.java | 48 -
.../beam/runners/flink/FlinkTestPipeline.java | 72 -
.../beam/runners/flink/PipelineOptionsTest.java | 184 -
.../beam/runners/flink/ReadSourceITCase.java | 85 -
.../flink/ReadSourceStreamingITCase.java | 74 -
.../beam/runners/flink/WriteSinkITCase.java | 171 -
.../flink/streaming/DoFnOperatorTest.java | 594 ----
.../FlinkBroadcastStateInternalsTest.java | 245 --
.../FlinkKeyGroupStateInternalsTest.java | 262 --
.../streaming/FlinkSplitStateInternalsTest.java | 101 -
.../streaming/FlinkStateInternalsTest.java | 395 ---
.../flink/streaming/GroupByNullKeyTest.java | 124 -
.../flink/streaming/TestCountingSource.java | 254 --
.../streaming/TopWikipediaSessionsITCase.java | 133 -
.../streaming/UnboundedSourceWrapperTest.java | 464 ---
.../runners/flink/streaming/package-info.java | 22 -
.../src/test/resources/log4j-test.properties | 27 -
.../flink/DefaultParallelismFactory.java | 39 +
.../flink/FlinkBatchPipelineTranslator.java | 139 +
.../flink/FlinkBatchTransformTranslators.java | 723 ++++
.../flink/FlinkBatchTranslationContext.java | 153 +
.../flink/FlinkDetachedRunnerResult.java | 75 +
.../FlinkPipelineExecutionEnvironment.java | 241 ++
.../runners/flink/FlinkPipelineOptions.java | 101 +
.../runners/flink/FlinkPipelineTranslator.java | 53 +
.../apache/beam/runners/flink/FlinkRunner.java | 232 ++
.../runners/flink/FlinkRunnerRegistrar.java | 62 +
.../beam/runners/flink/FlinkRunnerResult.java | 98 +
.../flink/FlinkStreamingPipelineTranslator.java | 276 ++
.../FlinkStreamingTransformTranslators.java | 1044 ++++++
.../flink/FlinkStreamingTranslationContext.java | 130 +
.../flink/FlinkStreamingViewOverrides.java | 372 ++
.../flink/PipelineTranslationOptimizer.java | 72 +
.../beam/runners/flink/TestFlinkRunner.java | 84 +
.../beam/runners/flink/TranslationMode.java | 31 +
.../apache/beam/runners/flink/package-info.java | 22 +
.../functions/FlinkAggregatorFactory.java | 53 +
.../functions/FlinkAssignContext.java | 63 +
.../functions/FlinkAssignWindows.java | 49 +
.../functions/FlinkDoFnFunction.java | 161 +
.../FlinkMergingNonShuffleReduceFunction.java | 228 ++
.../FlinkMergingPartialReduceFunction.java | 201 ++
.../functions/FlinkMergingReduceFunction.java | 199 ++
.../FlinkMultiOutputPruningFunction.java | 50 +
.../functions/FlinkNoOpStepContext.java | 73 +
.../functions/FlinkPartialReduceFunction.java | 172 +
.../functions/FlinkReduceFunction.java | 173 +
.../functions/FlinkSideInputReader.java | 80 +
.../functions/FlinkStatefulDoFnFunction.java | 198 ++
.../functions/SideInputInitializer.java | 73 +
.../translation/functions/package-info.java | 22 +
.../runners/flink/translation/package-info.java | 22 +
.../translation/types/CoderTypeInformation.java | 120 +
.../translation/types/CoderTypeSerializer.java | 132 +
.../types/EncodedValueComparator.java | 195 ++
.../types/EncodedValueSerializer.java | 113 +
.../types/EncodedValueTypeInformation.java | 98 +
.../types/InspectableByteArrayOutputStream.java | 34 +
.../flink/translation/types/KvKeySelector.java | 50 +
.../flink/translation/types/package-info.java | 22 +
.../utils/SerializedPipelineOptions.java | 67 +
.../flink/translation/utils/package-info.java | 22 +
.../wrappers/DataInputViewWrapper.java | 58 +
.../wrappers/DataOutputViewWrapper.java | 51 +
.../SerializableFnAggregatorWrapper.java | 98 +
.../translation/wrappers/SourceInputFormat.java | 150 +
.../translation/wrappers/SourceInputSplit.java | 52 +
.../translation/wrappers/package-info.java | 22 +
.../wrappers/streaming/DoFnOperator.java | 774 ++++
.../streaming/KvToByteBufferKeySelector.java | 56 +
.../streaming/SingletonKeyedWorkItem.java | 56 +
.../streaming/SingletonKeyedWorkItemCoder.java | 126 +
.../streaming/SplittableDoFnOperator.java | 150 +
.../wrappers/streaming/WindowDoFnOperator.java | 117 +
.../wrappers/streaming/WorkItemKeySelector.java | 56 +
.../streaming/io/BoundedSourceWrapper.java | 218 ++
.../streaming/io/UnboundedSocketSource.java | 249 ++
.../streaming/io/UnboundedSourceWrapper.java | 476 +++
.../wrappers/streaming/io/package-info.java | 22 +
.../wrappers/streaming/package-info.java | 22 +
.../state/FlinkBroadcastStateInternals.java | 865 +++++
.../state/FlinkKeyGroupStateInternals.java | 487 +++
.../state/FlinkSplitStateInternals.java | 260 ++
.../streaming/state/FlinkStateInternals.java | 1053 ++++++
.../state/KeyGroupCheckpointedOperator.java | 35 +
.../state/KeyGroupRestoringOperator.java | 32 +
.../wrappers/streaming/state/package-info.java | 22 +
.../flink/src/main/resources/log4j.properties | 23 +
.../flink/EncodedValueComparatorTest.java | 70 +
.../runners/flink/FlinkRunnerRegistrarTest.java | 48 +
.../beam/runners/flink/FlinkTestPipeline.java | 72 +
.../beam/runners/flink/PipelineOptionsTest.java | 184 +
.../beam/runners/flink/ReadSourceITCase.java | 85 +
.../flink/ReadSourceStreamingITCase.java | 74 +
.../beam/runners/flink/WriteSinkITCase.java | 192 +
.../flink/streaming/DoFnOperatorTest.java | 600 ++++
.../FlinkBroadcastStateInternalsTest.java | 245 ++
.../FlinkKeyGroupStateInternalsTest.java | 262 ++
.../streaming/FlinkSplitStateInternalsTest.java | 101 +
.../streaming/FlinkStateInternalsTest.java | 395 +++
.../flink/streaming/GroupByNullKeyTest.java | 124 +
.../flink/streaming/TestCountingSource.java | 254 ++
.../streaming/TopWikipediaSessionsITCase.java | 133 +
.../streaming/UnboundedSourceWrapperTest.java | 464 +++
.../runners/flink/streaming/package-info.java | 22 +
.../src/test/resources/log4j-test.properties | 27 +
.../translators/ParDoBoundMultiTranslator.java | 104 -
.../translators/ParDoBoundTranslator.java | 75 -
.../translators/ParDoMultiOutputTranslator.java | 104 +
.../ParDoSingleOutputTranslator.java | 75 +
runners/google-cloud-dataflow-java/pom.xml | 88 +-
.../beam/runners/dataflow/AssignWindows.java | 4 +-
.../dataflow/BatchStatefulParDoOverrides.java | 78 +-
.../runners/dataflow/BatchViewOverrides.java | 85 +-
.../runners/dataflow/CreateDataflowView.java | 46 +
.../beam/runners/dataflow/DataflowMetrics.java | 225 ++
.../runners/dataflow/DataflowPipelineJob.java | 96 +-
.../dataflow/DataflowPipelineTranslator.java | 71 +-
.../beam/runners/dataflow/DataflowRunner.java | 387 +-
.../runners/dataflow/DataflowRunnerInfo.java | 38 +-
.../DataflowUnboundedReadFromBoundedSource.java | 547 ---
.../dataflow/PrimitiveParDoSingleFactory.java | 85 +
.../dataflow/ReshuffleOverrideFactory.java | 90 +
.../dataflow/StreamingViewOverrides.java | 17 +-
.../runners/dataflow/TransformTranslator.java | 6 +-
.../dataflow/internal/CustomSources.java | 2 +-
.../runners/dataflow/internal/IsmFormat.java | 3 +-
.../options/DataflowPipelineDebugOptions.java | 4 +-
.../DataflowPipelineWorkerPoolOptions.java | 8 +-
.../dataflow/testing/TestDataflowRunner.java | 43 +-
.../runners/dataflow/testing/package-info.java | 2 +-
.../runners/dataflow/util/MonitoringUtil.java | 4 +
.../beam/runners/dataflow/dataflow.properties | 8 +-
.../BatchStatefulParDoOverridesTest.java | 2 +-
.../dataflow/BatchViewOverridesTest.java | 4 +-
.../runners/dataflow/DataflowMetricsTest.java | 236 ++
.../dataflow/DataflowPipelineJobTest.java | 50 +-
.../DataflowPipelineTranslatorTest.java | 85 +-
.../dataflow/DataflowRunnerInfoTest.java | 23 +-
.../runners/dataflow/DataflowRunnerTest.java | 98 +-
...aflowUnboundedReadFromBoundedSourceTest.java | 79 -
.../PrimitiveParDoSingleFactoryTest.java | 161 +
.../testing/TestDataflowRunnerTest.java | 3 +-
.../dataflow/util/MonitoringUtilTest.java | 31 +-
runners/pom.xml | 40 -
runners/spark/pom.xml | 78 +-
.../spark/SparkNativePipelineVisitor.java | 1 -
.../beam/runners/spark/SparkPipelineResult.java | 81 +-
.../apache/beam/runners/spark/SparkRunner.java | 195 +-
.../beam/runners/spark/TestSparkRunner.java | 45 +-
.../spark/coders/StatelessJavaSerializer.java | 11 +
.../beam/runners/spark/io/MicrobatchSource.java | 118 +-
.../beam/runners/spark/io/SourceDStream.java | 63 +-
.../apache/beam/runners/spark/io/SourceRDD.java | 74 +-
.../runners/spark/io/SparkUnboundedSource.java | 132 +-
.../beam/runners/spark/io/hadoop/HadoopIO.java | 216 --
.../spark/io/hadoop/ShardNameBuilder.java | 111 -
.../spark/io/hadoop/ShardNameTemplateAware.java | 31 -
.../io/hadoop/ShardNameTemplateHelper.java | 63 -
.../io/hadoop/TemplatedAvroKeyOutputFormat.java | 45 -
.../TemplatedSequenceFileOutputFormat.java | 45 -
.../io/hadoop/TemplatedTextOutputFormat.java | 45 -
.../runners/spark/io/hadoop/package-info.java | 22 -
.../runners/spark/metrics/SparkBeamMetric.java | 4 +
.../spark/metrics/SparkMetricResults.java | 67 +-
.../spark/metrics/SparkMetricsContainer.java | 31 +-
.../SparkGroupAlsoByWindowViaWindowSet.java | 48 +-
.../spark/stateful/SparkStateInternals.java | 30 +-
.../spark/stateful/StateSpecFunctions.java | 61 +-
.../spark/translation/BoundedDataset.java | 3 +-
.../runners/spark/translation/DoFnFunction.java | 130 -
.../spark/translation/EvaluationContext.java | 72 +-
.../translation/GroupCombineFunctions.java | 53 +-
.../spark/translation/MultiDoFnFunction.java | 4 +-
...SparkGroupAlsoByWindowViaOutputBufferFn.java | 10 +-
.../spark/translation/SparkProcessContext.java | 2 +-
.../spark/translation/SparkRuntimeContext.java | 2 +
.../spark/translation/TransformTranslator.java | 370 +-
.../spark/translation/TranslationUtils.java | 245 +-
.../SparkRunnerStreamingContextFactory.java | 45 +-
.../streaming/StreamingTransformTranslator.java | 158 +-
.../spark/util/GlobalWatermarkHolder.java | 2 +-
.../spark/util/SparkSideInputReader.java | 3 +-
.../apache/beam/runners/spark/CacheTest.java | 61 +
.../runners/spark/SparkPipelineStateTest.java | 2 +-
.../runners/spark/UsesCheckpointRecovery.java | 23 +
.../io/hadoop/HadoopFileFormatPipelineTest.java | 121 -
.../spark/io/hadoop/ShardNameBuilderTest.java | 88 -
.../spark/translation/StorageLevelTest.java | 6 +-
.../translation/streaming/CreateStreamTest.java | 4 +-
.../ResumeFromCheckpointStreamingTest.java | 17 +-
.../streaming/StreamingSourceMetricsTest.java | 71 +
.../streaming/TrackStreamingSourcesTest.java | 4 +-
runners/spark/src/test/resources/test_text.txt | 2 -
sdks/common/fn-api/pom.xml | 5 -
.../fn-api/src/main/proto/beam_fn_api.proto | 174 +-
.../src/main/proto/beam_runner_api.proto | 60 +-
sdks/java/build-tools/pom.xml | 31 +-
.../src/main/resources/beam/beam-codestyle.xml | 329 ++
.../src/main/resources/beam/findbugs-filter.xml | 43 +-
.../beam/codestyle/CodeStyleTestClass.java | 45 +
sdks/java/core/pom.xml | 155 +-
.../beam/sdk/AggregatorPipelineExtractor.java | 8 +-
.../main/java/org/apache/beam/sdk/Pipeline.java | 123 +-
.../java/org/apache/beam/sdk/io/AvroIO.java | 170 +-
.../java/org/apache/beam/sdk/io/AvroSource.java | 4 +-
.../sdk/io/BoundedReadFromUnboundedSource.java | 4 +-
.../org/apache/beam/sdk/io/BoundedSource.java | 20 +-
.../apache/beam/sdk/io/CompressedSource.java | 17 +-
.../org/apache/beam/sdk/io/CountingInput.java | 4 +
.../org/apache/beam/sdk/io/CountingSource.java | 10 +-
.../org/apache/beam/sdk/io/FileBasedSink.java | 429 ++-
.../org/apache/beam/sdk/io/FileBasedSource.java | 38 +-
.../org/apache/beam/sdk/io/FileSystems.java | 302 +-
.../org/apache/beam/sdk/io/LocalFileSystem.java | 57 +-
.../org/apache/beam/sdk/io/LocalResourceId.java | 2 +-
.../apache/beam/sdk/io/OffsetBasedSource.java | 4 +-
.../java/org/apache/beam/sdk/io/PubsubIO.java | 1201 -------
.../apache/beam/sdk/io/PubsubUnboundedSink.java | 500 ---
.../beam/sdk/io/PubsubUnboundedSource.java | 1435 --------
.../main/java/org/apache/beam/sdk/io/Sink.java | 57 +-
.../java/org/apache/beam/sdk/io/TFRecordIO.java | 694 ++++
.../java/org/apache/beam/sdk/io/TextIO.java | 104 +-
.../org/apache/beam/sdk/io/UnboundedSource.java | 19 +-
.../main/java/org/apache/beam/sdk/io/Write.java | 377 +-
.../main/java/org/apache/beam/sdk/io/XmlIO.java | 477 +++
.../java/org/apache/beam/sdk/io/XmlSink.java | 226 +-
.../java/org/apache/beam/sdk/io/XmlSource.java | 191 +-
.../org/apache/beam/sdk/io/fs/MoveOptions.java | 34 +
.../org/apache/beam/sdk/io/fs/ResourceId.java | 8 +
.../org/apache/beam/sdk/io/package-info.java | 2 +-
.../apache/beam/sdk/io/range/ByteKeyRange.java | 4 +-
.../java/org/apache/beam/sdk/metrics/Gauge.java | 32 +
.../org/apache/beam/sdk/metrics/GaugeCell.java | 60 +
.../org/apache/beam/sdk/metrics/GaugeData.java | 81 +
.../apache/beam/sdk/metrics/GaugeResult.java | 61 +
.../beam/sdk/metrics/MetricFiltering.java | 99 +
.../beam/sdk/metrics/MetricNameFilter.java | 3 +-
.../beam/sdk/metrics/MetricQueryResults.java | 3 +
.../apache/beam/sdk/metrics/MetricUpdates.java | 11 +-
.../org/apache/beam/sdk/metrics/Metrics.java | 35 +
.../beam/sdk/metrics/MetricsContainer.java | 26 +-
.../beam/sdk/options/BigQueryOptions.java | 32 -
.../options/CloudResourceManagerOptions.java | 40 -
.../DefaultPipelineOptionsRegistrar.java | 5 -
.../org/apache/beam/sdk/options/GcpOptions.java | 227 --
.../org/apache/beam/sdk/options/GcsOptions.java | 158 -
.../beam/sdk/options/GoogleApiDebugOptions.java | 87 -
.../beam/sdk/options/PipelineOptions.java | 16 +-
.../apache/beam/sdk/options/PubsubOptions.java | 36 -
.../beam/sdk/options/StreamingOptions.java | 7 +-
.../beam/sdk/runners/PTransformOverride.java | 44 +
.../sdk/runners/PTransformOverrideFactory.java | 31 +-
.../apache/beam/sdk/runners/PipelineRunner.java | 7 +-
.../beam/sdk/runners/TransformHierarchy.java | 80 +-
.../beam/sdk/testing/BigqueryMatcher.java | 256 --
.../apache/beam/sdk/testing/GatherAllPanes.java | 1 +
.../org/apache/beam/sdk/testing/PAssert.java | 220 +-
.../apache/beam/sdk/testing/PaneExtractors.java | 1 +
.../beam/sdk/testing/RunnableOnService.java | 38 -
.../beam/sdk/testing/SerializableMatcher.java | 2 +-
.../beam/sdk/testing/SerializableMatchers.java | 4 +-
.../beam/sdk/testing/SourceTestUtils.java | 6 +-
.../apache/beam/sdk/testing/StaticWindows.java | 17 +-
.../apache/beam/sdk/testing/StreamingIT.java | 2 +-
.../apache/beam/sdk/testing/TestPipeline.java | 89 +-
.../beam/sdk/testing/TestPipelineOptions.java | 5 +
...esSplittableParDoWithWindowedSideInputs.java | 26 +
.../beam/sdk/testing/ValidatesRunner.java | 23 +
.../beam/sdk/testing/ValueInSingleWindow.java | 134 -
.../beam/sdk/transforms/AppliedPTransform.java | 24 +-
.../org/apache/beam/sdk/transforms/Combine.java | 64 +-
.../org/apache/beam/sdk/transforms/Create.java | 86 +-
.../org/apache/beam/sdk/transforms/DoFn.java | 138 +-
.../apache/beam/sdk/transforms/DoFnTester.java | 51 +-
.../beam/sdk/transforms/FlatMapElements.java | 113 +-
.../apache/beam/sdk/transforms/GroupByKey.java | 6 +-
.../beam/sdk/transforms/GroupIntoBatches.java | 229 ++
.../apache/beam/sdk/transforms/MapElements.java | 99 +-
.../beam/sdk/transforms/Materialization.java | 36 +
.../beam/sdk/transforms/Materializations.java | 53 +
.../apache/beam/sdk/transforms/PTransform.java | 16 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 495 +--
.../apache/beam/sdk/transforms/Partition.java | 6 +-
.../org/apache/beam/sdk/transforms/Regex.java | 14 +-
.../org/apache/beam/sdk/transforms/Sample.java | 11 +-
.../org/apache/beam/sdk/transforms/View.java | 81 +-
.../org/apache/beam/sdk/transforms/ViewFn.java | 5 +
.../beam/sdk/transforms/WithTimestamps.java | 26 +-
.../sdk/transforms/display/DisplayData.java | 6 +
.../transforms/join/KeyedPCollectionTuple.java | 12 +-
.../reflect/ByteBuddyDoFnInvokerFactory.java | 47 +-
.../sdk/transforms/reflect/DoFnInvoker.java | 4 +-
.../sdk/transforms/reflect/DoFnSignature.java | 10 +-
.../sdk/transforms/reflect/DoFnSignatures.java | 96 +-
.../splittabledofn/HasDefaultTracker.java | 30 +
.../transforms/splittabledofn/OffsetRange.java | 8 +-
.../splittabledofn/OffsetRangeTracker.java | 33 +-
.../splittabledofn/RestrictionTracker.java | 8 +
.../transforms/windowing/AfterWatermark.java | 16 +-
.../sdk/transforms/windowing/GlobalWindow.java | 10 +
.../sdk/transforms/windowing/GlobalWindows.java | 11 +-
.../transforms/windowing/InvalidWindows.java | 2 +-
.../beam/sdk/transforms/windowing/PaneInfo.java | 4 +-
.../windowing/PartitioningWindowFn.java | 17 +-
.../sdk/transforms/windowing/Repeatedly.java | 2 +-
.../beam/sdk/transforms/windowing/Sessions.java | 2 +-
.../transforms/windowing/SlidingWindows.java | 27 +-
.../beam/sdk/transforms/windowing/Triggers.java | 320 --
.../beam/sdk/transforms/windowing/Window.java | 515 ++-
.../beam/sdk/transforms/windowing/WindowFn.java | 9 +-
.../transforms/windowing/WindowMappingFn.java | 67 +
.../beam/sdk/util/AppEngineEnvironment.java | 62 -
...AttemptAndTimeBoundedExponentialBackOff.java | 170 -
.../util/AttemptBoundedExponentialBackOff.java | 86 -
.../org/apache/beam/sdk/util/CoderUtils.java | 21 +-
.../beam/sdk/util/CombineContextFactory.java | 2 +-
.../apache/beam/sdk/util/CredentialFactory.java | 29 -
.../org/apache/beam/sdk/util/DefaultBucket.java | 105 -
.../beam/sdk/util/DirectSideInputReader.java | 74 -
.../beam/sdk/util/FileIOChannelFactory.java | 23 +-
.../beam/sdk/util/GcpCredentialFactory.java | 67 -
.../apache/beam/sdk/util/GcpProjectUtil.java | 106 -
.../beam/sdk/util/GcsIOChannelFactory.java | 110 -
.../sdk/util/GcsIOChannelFactoryRegistrar.java | 38 -
.../apache/beam/sdk/util/GcsPathValidator.java | 95 -
.../java/org/apache/beam/sdk/util/GcsUtil.java | 803 -----
.../apache/beam/sdk/util/IOChannelFactory.java | 3 +-
.../apache/beam/sdk/util/IOChannelUtils.java | 22 +-
.../apache/beam/sdk/util/IdentityWindowFn.java | 3 +-
.../util/IntervalBoundedExponentialBackOff.java | 89 -
.../apache/beam/sdk/util/MutationDetector.java | 2 +-
.../org/apache/beam/sdk/util/NameUtils.java | 5 +
.../beam/sdk/util/NoopCredentialFactory.java | 68 -
.../sdk/util/NullCredentialInitializer.java | 62 -
.../beam/sdk/util/NumberedShardedFile.java | 30 +-
.../apache/beam/sdk/util/PCollectionViews.java | 268 +-
.../java/org/apache/beam/sdk/util/PTuple.java | 160 -
.../org/apache/beam/sdk/util/PubsubClient.java | 544 ---
.../apache/beam/sdk/util/PubsubGrpcClient.java | 424 ---
.../apache/beam/sdk/util/PubsubJsonClient.java | 317 --
.../apache/beam/sdk/util/PubsubTestClient.java | 436 ---
.../apache/beam/sdk/util/ReifyTimestamps.java | 76 +
.../org/apache/beam/sdk/util/Reshuffle.java | 21 +-
.../apache/beam/sdk/util/TestCredential.java | 59 -
.../org/apache/beam/sdk/util/TimeDomain.java | 2 +-
.../org/apache/beam/sdk/util/Transport.java | 179 -
.../beam/sdk/util/WindowingStrategies.java | 266 --
.../util/state/AccumulatorCombiningState.java | 53 -
.../apache/beam/sdk/util/state/BagState.java | 2 +-
.../beam/sdk/util/state/CombiningState.java | 27 +-
.../beam/sdk/util/state/GroupingState.java | 42 +
.../apache/beam/sdk/util/state/MapState.java | 52 +-
.../beam/sdk/util/state/ReadableStates.java | 45 +
.../apache/beam/sdk/util/state/SetState.java | 36 +-
.../org/apache/beam/sdk/util/state/State.java | 2 +-
.../apache/beam/sdk/util/state/StateBinder.java | 12 +-
.../apache/beam/sdk/util/state/StateSpec.java | 15 +
.../apache/beam/sdk/util/state/StateSpecs.java | 344 +-
.../beam/sdk/util/state/WatermarkHoldState.java | 2 +-
.../java/org/apache/beam/sdk/values/PBegin.java | 6 +-
.../apache/beam/sdk/values/PCollectionList.java | 27 +-
.../beam/sdk/values/PCollectionTuple.java | 13 +-
.../apache/beam/sdk/values/PCollectionView.java | 19 +
.../java/org/apache/beam/sdk/values/PDone.java | 6 +-
.../java/org/apache/beam/sdk/values/PInput.java | 4 +-
.../org/apache/beam/sdk/values/POutput.java | 4 +-
.../beam/sdk/values/POutputValueBase.java | 41 +-
.../java/org/apache/beam/sdk/values/PValue.java | 4 +-
.../org/apache/beam/sdk/values/PValueBase.java | 6 +-
.../apache/beam/sdk/values/TaggedPValue.java | 5 +
.../org/apache/beam/sdk/values/TupleTag.java | 26 +-
.../apache/beam/sdk/values/TupleTagList.java | 2 +-
.../apache/beam/sdk/values/TypeDescriptors.java | 25 +-
.../org/apache/beam/sdk/values/TypedPValue.java | 6 +-
.../beam/sdk/values/ValueInSingleWindow.java | 134 +
sdks/java/core/src/main/proto/README.md | 3 -
.../org/apache/beam/SdkCoreApiSurfaceTest.java | 3 -
.../sdk/AggregatorPipelineExtractorTest.java | 56 +-
.../java/org/apache/beam/sdk/PipelineTest.java | 218 +-
.../sdk/coders/protobuf/ProtobufUtilTest.java | 1 -
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 154 +-
.../org/apache/beam/sdk/io/AvroSourceTest.java | 10 +-
.../io/BoundedReadFromUnboundedSourceTest.java | 8 +-
.../apache/beam/sdk/io/CountingInputTest.java | 14 +-
.../apache/beam/sdk/io/CountingSourceTest.java | 20 +-
.../apache/beam/sdk/io/FileBasedSinkTest.java | 94 +-
.../apache/beam/sdk/io/FileBasedSourceTest.java | 8 +-
.../org/apache/beam/sdk/io/FileSystemsTest.java | 168 +-
.../apache/beam/sdk/io/LocalFileSystemTest.java | 12 +-
.../beam/sdk/io/OffsetBasedSourceTest.java | 8 +-
.../org/apache/beam/sdk/io/PubsubIOTest.java | 197 --
.../beam/sdk/io/PubsubUnboundedSinkTest.java | 190 -
.../beam/sdk/io/PubsubUnboundedSourceTest.java | 398 ---
.../java/org/apache/beam/sdk/io/ReadTest.java | 10 +-
.../org/apache/beam/sdk/io/TFRecordIOTest.java | 369 ++
.../java/org/apache/beam/sdk/io/TextIOTest.java | 125 +-
.../java/org/apache/beam/sdk/io/WriteTest.java | 53 +-
.../org/apache/beam/sdk/io/XmlSinkTest.java | 89 +-
.../org/apache/beam/sdk/io/XmlSourceTest.java | 250 +-
.../apache/beam/sdk/metrics/GaugeCellTest.java | 48 +
.../beam/sdk/metrics/MetricFilteringTest.java | 145 +
.../apache/beam/sdk/metrics/MetricMatchers.java | 12 +-
.../apache/beam/sdk/metrics/MetricsTest.java | 93 +-
.../apache/beam/sdk/options/GcpOptionsTest.java | 171 -
.../sdk/options/GoogleApiDebugOptionsTest.java | 145 -
.../sdk/options/PipelineOptionsFactoryTest.java | 4 +-
.../beam/sdk/runners/PipelineRunnerTest.java | 46 +-
.../sdk/runners/TransformHierarchyTest.java | 86 +-
.../beam/sdk/runners/TransformTreeTest.java | 88 +-
.../runners/dataflow/TestCountingSource.java | 2 +-
.../beam/sdk/testing/BigqueryMatcherTest.java | 176 -
.../beam/sdk/testing/GatherAllPanesTest.java | 1 +
.../apache/beam/sdk/testing/PAssertTest.java | 74 +-
.../sdk/testing/PCollectionViewTesting.java | 83 +-
.../beam/sdk/testing/PaneExtractorsTest.java | 1 +
.../beam/sdk/testing/SourceTestUtilsTest.java | 2 +-
.../beam/sdk/testing/StaticWindowsTest.java | 10 +-
.../beam/sdk/testing/TestPipelineTest.java | 47 +-
.../apache/beam/sdk/testing/TestStreamTest.java | 6 +-
.../testing/ValueInSingleWindowCoderTest.java | 1 +
.../sdk/transforms/ApproximateUniqueTest.java | 4 +-
.../beam/sdk/transforms/CombineFnsTest.java | 8 +-
.../apache/beam/sdk/transforms/CombineTest.java | 44 +-
.../apache/beam/sdk/transforms/CountTest.java | 10 +-
.../apache/beam/sdk/transforms/CreateTest.java | 70 +-
.../beam/sdk/transforms/DistinctTest.java | 8 +-
.../beam/sdk/transforms/DoFnTesterTest.java | 7 +-
.../apache/beam/sdk/transforms/FilterTest.java | 16 +-
.../apache/beam/sdk/transforms/FlattenTest.java | 47 +-
.../beam/sdk/transforms/GroupByKeyTest.java | 14 +-
.../sdk/transforms/GroupIntoBatchesTest.java | 232 ++
.../apache/beam/sdk/transforms/KeysTest.java | 6 +-
.../apache/beam/sdk/transforms/KvSwapTest.java | 6 +-
.../beam/sdk/transforms/MapElementsTest.java | 29 +-
.../beam/sdk/transforms/ParDoLifecycleTest.java | 10 +-
.../apache/beam/sdk/transforms/ParDoTest.java | 1180 +++++--
.../beam/sdk/transforms/PartitionTest.java | 4 +-
.../apache/beam/sdk/transforms/SampleTest.java | 14 +-
.../beam/sdk/transforms/SplittableDoFnTest.java | 157 +-
.../beam/sdk/transforms/ToStringTest.java | 13 +-
.../org/apache/beam/sdk/transforms/TopTest.java | 3 +-
.../apache/beam/sdk/transforms/ValuesTest.java | 6 +-
.../apache/beam/sdk/transforms/ViewTest.java | 222 +-
.../beam/sdk/transforms/WithTimestampsTest.java | 8 +-
.../sdk/transforms/display/DisplayDataTest.java | 17 +
.../sdk/transforms/join/CoGroupByKeyTest.java | 10 +-
.../transforms/reflect/DoFnInvokersTest.java | 153 +-
.../DoFnSignaturesProcessElementTest.java | 2 +-
.../DoFnSignaturesSplittableDoFnTest.java | 117 +-
.../splittabledofn/OffsetRangeTrackerTest.java | 49 +-
.../windowing/CalendarWindowsTest.java | 33 +
.../transforms/windowing/FixedWindowsTest.java | 33 +
.../windowing/SlidingWindowsTest.java | 16 +-
.../sdk/transforms/windowing/TriggersTest.java | 100 -
.../sdk/transforms/windowing/WindowTest.java | 97 +-
.../sdk/transforms/windowing/WindowingTest.java | 10 +-
...mptAndTimeBoundedExponentialBackOffTest.java | 213 --
.../AttemptBoundedExponentialBackOffTest.java | 85 -
.../apache/beam/sdk/util/DefaultBucketTest.java | 112 -
.../beam/sdk/util/GcpProjectUtilTest.java | 76 -
.../util/GcsIOChannelFactoryRegistrarTest.java | 44 -
.../beam/sdk/util/GcsIOChannelFactoryTest.java | 43 -
.../beam/sdk/util/GcsPathValidatorTest.java | 87 -
.../org/apache/beam/sdk/util/GcsUtilTest.java | 798 -----
.../sdk/util/IdentitySideInputWindowFn.java | 10 +-
.../IntervalBoundedExponentialBackOffTest.java | 100 -
.../org/apache/beam/sdk/util/NameUtilsTest.java | 12 +
.../org/apache/beam/sdk/util/PTupleTest.java | 40 -
.../apache/beam/sdk/util/PubsubClientTest.java | 189 -
.../beam/sdk/util/PubsubGrpcClientTest.java | 207 --
.../beam/sdk/util/PubsubJsonClientTest.java | 133 -
.../beam/sdk/util/PubsubTestClientTest.java | 114 -
.../beam/sdk/util/ReifyTimestampsTest.java | 109 +
.../org/apache/beam/sdk/util/ReshuffleTest.java | 84 +-
.../util/RetryHttpRequestInitializerTest.java | 290 --
.../beam/sdk/util/WindowingStrategiesTest.java | 91 -
.../beam/sdk/values/PCollectionListTest.java | 70 +-
.../beam/sdk/values/PCollectionTupleTest.java | 17 +-
.../org/apache/beam/sdk/values/PDoneTest.java | 4 +-
.../apache/beam/sdk/values/TypedPValueTest.java | 50 +-
sdks/java/extensions/gcp-core/pom.xml | 222 ++
.../beam/sdk/options/BigQueryOptions.java | 32 +
.../options/CloudResourceManagerOptions.java | 40 +
.../org/apache/beam/sdk/options/GcpOptions.java | 227 ++
.../options/GcpPipelineOptionsRegistrar.java | 39 +
.../org/apache/beam/sdk/options/GcsOptions.java | 154 +
.../beam/sdk/options/GoogleApiDebugOptions.java | 87 +
.../apache/beam/sdk/options/PubsubOptions.java | 36 +
.../apache/beam/sdk/options/package-info.java | 22 +
.../beam/sdk/testing/BigqueryMatcher.java | 256 ++
.../apache/beam/sdk/testing/package-info.java | 21 +
.../apache/beam/sdk/util/CredentialFactory.java | 29 +
.../org/apache/beam/sdk/util/DefaultBucket.java | 105 +
.../beam/sdk/util/GcpCredentialFactory.java | 67 +
.../apache/beam/sdk/util/GcpProjectUtil.java | 106 +
.../beam/sdk/util/GcsIOChannelFactory.java | 111 +
.../sdk/util/GcsIOChannelFactoryRegistrar.java | 38 +
.../apache/beam/sdk/util/GcsPathValidator.java | 95 +
.../java/org/apache/beam/sdk/util/GcsUtil.java | 798 +++++
.../beam/sdk/util/NoopCredentialFactory.java | 68 +
.../sdk/util/NullCredentialInitializer.java | 62 +
.../apache/beam/sdk/util/TestCredential.java | 59 +
.../org/apache/beam/sdk/util/Transport.java | 178 +
.../org/apache/beam/sdk/util/package-info.java | 20 +
.../org/apache/beam/GcpCoreApiSurfaceTest.java | 62 +
.../apache/beam/sdk/options/GcpOptionsTest.java | 171 +
.../sdk/options/GoogleApiDebugOptionsTest.java | 145 +
.../beam/sdk/testing/BigqueryMatcherTest.java | 176 +
.../apache/beam/sdk/util/DefaultBucketTest.java | 112 +
.../beam/sdk/util/GcpProjectUtilTest.java | 76 +
.../util/GcsIOChannelFactoryRegistrarTest.java | 44 +
.../beam/sdk/util/GcsIOChannelFactoryTest.java | 43 +
.../beam/sdk/util/GcsPathValidatorTest.java | 87 +
.../org/apache/beam/sdk/util/GcsUtilTest.java | 798 +++++
.../util/RetryHttpRequestInitializerTest.java | 290 ++
sdks/java/extensions/jackson/pom.xml | 40 -
sdks/java/extensions/pom.xml | 1 +
sdks/java/extensions/sorter/pom.xml | 49 -
sdks/java/harness/pom.xml | 10 +-
.../beam/fn/harness/fake/FakeStepContext.java | 2 +-
.../control/ProcessBundleHandlerTest.java | 30 +-
sdks/java/io/common/pom.xml | 42 +
.../apache/beam/sdk/io/common/HashingFn.java | 109 +
.../sdk/io/common/IOTestPipelineOptions.java | 79 +
sdks/java/io/elasticsearch/pom.xml | 7 +
.../sdk/io/elasticsearch/ElasticsearchIO.java | 37 +-
.../sdk/io/elasticsearch/ElasticsearchIOIT.java | 9 +-
.../io/elasticsearch/ElasticsearchIOTest.java | 15 +-
.../elasticsearch/ElasticsearchTestDataSet.java | 16 +-
.../elasticsearch/ElasticsearchTestOptions.java | 46 -
sdks/java/io/google-cloud-platform/pom.xml | 93 +-
.../beam/sdk/io/gcp/bigquery/BatchLoads.java | 225 ++
.../sdk/io/gcp/bigquery/BigQueryHelpers.java | 352 ++
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 3306 ++++--------------
.../io/gcp/bigquery/BigQueryQuerySource.java | 205 ++
.../io/gcp/bigquery/BigQueryServicesImpl.java | 2 +-
.../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 205 ++
.../gcp/bigquery/BigQueryTableRowIterator.java | 3 +-
.../io/gcp/bigquery/BigQueryTableSource.java | 132 +
.../beam/sdk/io/gcp/bigquery/CreateTables.java | 127 +
.../io/gcp/bigquery/GenerateShardedTable.java | 47 +
.../io/gcp/bigquery/PassThroughThenCleanup.java | 84 +
.../beam/sdk/io/gcp/bigquery/PrepareWrite.java | 81 +
.../beam/sdk/io/gcp/bigquery/ShardedKey.java | 67 +
.../sdk/io/gcp/bigquery/ShardedKeyCoder.java | 87 +
.../sdk/io/gcp/bigquery/StreamingInserts.java | 79 +
.../sdk/io/gcp/bigquery/StreamingWriteFn.java | 111 +
.../io/gcp/bigquery/StreamingWriteTables.java | 86 +
.../sdk/io/gcp/bigquery/TableDestination.java | 76 +
.../io/gcp/bigquery/TableDestinationCoder.java | 60 +
.../beam/sdk/io/gcp/bigquery/TableRowInfo.java | 34 +
.../sdk/io/gcp/bigquery/TableRowInfoCoder.java | 68 +
.../sdk/io/gcp/bigquery/TableRowWriter.java | 91 +
.../sdk/io/gcp/bigquery/TagWithUniqueIds.java | 62 +
.../sdk/io/gcp/bigquery/TransformingSource.java | 136 +
.../io/gcp/bigquery/WriteBundlesToFiles.java | 157 +
.../sdk/io/gcp/bigquery/WritePartition.java | 174 +
.../beam/sdk/io/gcp/bigquery/WriteRename.java | 185 +
.../beam/sdk/io/gcp/bigquery/WriteResult.java | 46 +
.../beam/sdk/io/gcp/bigquery/WriteTables.java | 211 ++
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 107 +-
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 398 ++-
.../beam/sdk/io/gcp/pubsub/PubsubClient.java | 544 +++
.../sdk/io/gcp/pubsub/PubsubGrpcClient.java | 424 +++
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 1014 ++++++
.../sdk/io/gcp/pubsub/PubsubJsonClient.java | 319 ++
.../sdk/io/gcp/pubsub/PubsubTestClient.java | 436 +++
.../sdk/io/gcp/pubsub/PubsubUnboundedSink.java | 490 +++
.../io/gcp/pubsub/PubsubUnboundedSource.java | 1463 ++++++++
.../beam/sdk/io/gcp/pubsub/package-info.java | 24 +
.../beam/sdk/io/gcp/storage/GcsResourceId.java | 2 +-
.../beam/sdk/io/gcp/GcpApiSurfaceTest.java | 8 +-
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 1812 ++++------
.../sdk/io/gcp/bigquery/BigQueryUtilTest.java | 12 +-
.../io/gcp/bigquery/FakeBigQueryServices.java | 166 +
.../sdk/io/gcp/bigquery/FakeDatasetService.java | 208 ++
.../sdk/io/gcp/bigquery/FakeJobService.java | 404 +++
.../sdk/io/gcp/bigquery/TableContainer.java | 61 +
.../sdk/io/gcp/bigtable/BigtableIOTest.java | 20 +-
.../sdk/io/gcp/bigtable/BigtableWriteIT.java | 15 +-
.../sdk/io/gcp/datastore/DatastoreV1Test.java | 252 +-
.../sdk/io/gcp/datastore/SplitQueryFnIT.java | 2 +-
.../beam/sdk/io/gcp/datastore/V1ReadIT.java | 66 +-
.../sdk/io/gcp/datastore/V1TestOptions.java | 2 +-
.../sdk/io/gcp/pubsub/PubsubClientTest.java | 189 +
.../sdk/io/gcp/pubsub/PubsubGrpcClientTest.java | 208 ++
.../beam/sdk/io/gcp/pubsub/PubsubIOTest.java | 189 +
.../sdk/io/gcp/pubsub/PubsubJsonClientTest.java | 139 +
.../sdk/io/gcp/pubsub/PubsubTestClientTest.java | 114 +
.../io/gcp/pubsub/PubsubUnboundedSinkTest.java | 188 +
.../gcp/pubsub/PubsubUnboundedSourceTest.java | 409 +++
.../io/hadoop/SerializableConfiguration.java | 3 +-
sdks/java/io/hadoop-input-format/README.md | 167 -
sdks/java/io/hadoop-input-format/pom.xml | 136 -
.../hadoop/inputformat/HadoopInputFormatIO.java | 941 -----
.../sdk/io/hadoop/inputformat/package-info.java | 23 -
.../ConfigurableEmployeeInputFormat.java | 131 -
.../sdk/io/hadoop/inputformat/Employee.java | 85 -
.../hadoop/inputformat/EmployeeInputFormat.java | 172 -
.../inputformat/HadoopInputFormatIOTest.java | 844 -----
.../ReuseObjectsEmployeeInputFormat.java | 176 -
.../hadoop/inputformat/TestEmployeeDataSet.java | 76 -
sdks/java/io/hadoop/README.md | 167 +
sdks/java/io/hadoop/input-format/pom.xml | 98 +
.../hadoop/inputformat/HadoopInputFormatIO.java | 844 +++++
.../sdk/io/hadoop/inputformat/package-info.java | 23 +
.../ConfigurableEmployeeInputFormat.java | 131 +
.../sdk/io/hadoop/inputformat/Employee.java | 85 +
.../hadoop/inputformat/EmployeeInputFormat.java | 172 +
.../inputformat/HadoopInputFormatIOTest.java | 851 +++++
.../ReuseObjectsEmployeeInputFormat.java | 176 +
.../hadoop/inputformat/TestEmployeeDataSet.java | 76 +
sdks/java/io/hadoop/jdk1.8-tests/pom.xml | 244 ++
.../inputformat/HIFIOWithElasticTest.java | 277 ++
.../HIFIOWithEmbeddedCassandraTest.java | 215 ++
.../custom/options/HIFTestOptions.java | 64 +
.../integration/tests/HIFIOCassandraIT.java | 173 +
.../integration/tests/HIFIOElasticIT.java | 215 ++
.../src/test/resources/cassandra.yaml | 1074 ++++++
sdks/java/io/hadoop/pom.xml | 64 +
sdks/java/io/hbase/pom.xml | 92 +-
.../org/apache/beam/sdk/io/hbase/HBaseIO.java | 7 +-
.../apache/beam/sdk/io/hbase/HBaseIOTest.java | 13 +-
.../src/test/resources/log4j-test.properties | 27 +
sdks/java/io/hdfs/README.md | 6 +-
sdks/java/io/hdfs/pom.xml | 41 -
.../apache/beam/sdk/io/hdfs/HDFSFileSink.java | 29 +-
.../apache/beam/sdk/io/hdfs/HDFSFileSource.java | 16 +-
.../beam/sdk/io/hdfs/HDFSFileSinkTest.java | 2 +-
.../beam/sdk/io/hdfs/HDFSFileSourceTest.java | 23 +-
sdks/java/io/jdbc/pom.xml | 53 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 75 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIOIT.java | 11 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 12 +-
.../beam/sdk/io/jdbc/JdbcTestDataSet.java | 12 +-
.../beam/sdk/io/jdbc/PostgresTestOptions.java | 60 -
.../kubernetes/postgres-pod-no-vol.yml | 32 -
.../kubernetes/postgres-service-public.yml | 28 -
.../jdbc/src/test/resources/kubernetes/setup.sh | 20 -
.../src/test/resources/kubernetes/teardown.sh | 20 -
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 4 +-
.../org/apache/beam/sdk/io/jms/JmsIOTest.java | 10 +-
sdks/java/io/kafka/pom.xml | 36 -
.../apache/beam/sdk/io/kafka/ConsumerSpEL.java | 43 +-
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 70 +-
.../apache/beam/sdk/io/kafka/KafkaRecord.java | 15 +-
.../beam/sdk/io/kafka/KafkaRecordCoder.java | 5 +
.../apache/beam/sdk/io/kafka/KafkaIOTest.java | 54 +-
.../apache/beam/sdk/io/kinesis/KinesisIO.java | 2 +
.../beam/sdk/io/kinesis/KinesisSource.java | 2 +-
.../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 14 +-
.../apache/beam/sdk/io/mongodb/MongoDbIO.java | 4 +-
.../sdk/io/mongodb/MongoDBGridFSIOTest.java | 7 +-
.../beam/sdk/io/mongodb/MongoDbIOTest.java | 5 -
.../org/apache/beam/sdk/io/mqtt/MqttIO.java | 4 +-
.../org/apache/beam/sdk/io/mqtt/MqttIOTest.java | 6 -
sdks/java/io/pom.xml | 3 +-
.../beam/sdk/transforms/FilterJava8Test.java | 8 +-
.../transforms/FlatMapElementsJava8Test.java | 10 +-
.../sdk/transforms/MapElementsJava8Test.java | 14 +-
.../beam/sdk/transforms/WithKeysJava8Test.java | 4 +-
.../sdk/transforms/WithTimestampsJava8Test.java | 4 +-
.../java/javadoc/dataflow-sdk-docs/package-list | 11 -
sdks/java/javadoc/overview.html | 2 +-
.../META-INF/maven/archetype-metadata.xml | 2 +-
.../META-INF/maven/archetype-metadata.xml | 2 +-
.../META-INF/maven/archetype-metadata.xml | 2 +-
sdks/python/.pylintrc | 6 +-
sdks/python/MANIFEST.in | 3 -
sdks/python/apache_beam/__init__.py | 7 +-
sdks/python/apache_beam/coders/coder_impl.py | 50 +-
sdks/python/apache_beam/coders/coders.py | 10 +-
sdks/python/apache_beam/coders/typecoders.py | 2 +-
sdks/python/apache_beam/error.py | 4 +
.../examples/complete/game/README.md | 69 +
.../examples/complete/game/__init__.py | 16 +
.../examples/complete/game/hourly_team_score.py | 294 ++
.../complete/game/hourly_team_score_test.py | 52 +
.../examples/complete/game/user_score.py | 217 ++
.../examples/complete/game/user_score_test.py | 49 +
.../examples/complete/top_wikipedia_sessions.py | 8 -
.../examples/cookbook/bigquery_tornadoes.py | 2 +-
.../cookbook/bigquery_tornadoes_it_test.py | 3 +
.../examples/cookbook/datastore_wordcount.py | 15 +-
.../examples/cookbook/group_with_coder.py | 6 +-
.../examples/cookbook/multiple_output_pardo.py | 47 +-
.../apache_beam/examples/snippets/snippets.py | 5 +-
.../examples/snippets/snippets_test.py | 36 +-
sdks/python/apache_beam/examples/wordcount.py | 48 +-
.../apache_beam/examples/wordcount_debugging.py | 4 -
.../apache_beam/examples/wordcount_it_test.py | 11 +-
sdks/python/apache_beam/internal/gcp/auth.py | 88 +-
.../apache_beam/internal/gcp/auth_test.py | 44 -
.../apache_beam/internal/gcp/json_value.py | 6 +
sdks/python/apache_beam/internal/pickler.py | 20 +-
sdks/python/apache_beam/io/__init__.py | 1 +
sdks/python/apache_beam/io/avroio.py | 12 +-
sdks/python/apache_beam/io/avroio_test.py | 14 +-
sdks/python/apache_beam/io/concat_source.py | 74 +-
.../python/apache_beam/io/concat_source_test.py | 12 +-
sdks/python/apache_beam/io/filebasedsource.py | 127 +-
.../apache_beam/io/filebasedsource_test.py | 43 +-
sdks/python/apache_beam/io/fileio.py | 638 +---
sdks/python/apache_beam/io/fileio_test.py | 166 +-
sdks/python/apache_beam/io/filesystem.py | 530 +++
sdks/python/apache_beam/io/filesystem_test.py | 213 ++
sdks/python/apache_beam/io/filesystems_util.py | 35 +
sdks/python/apache_beam/io/gcp/bigquery.py | 94 +-
sdks/python/apache_beam/io/gcp/bigquery_test.py | 3 +-
.../io/gcp/datastore/v1/datastoreio.py | 24 +-
.../io/gcp/datastore/v1/datastoreio_test.py | 4 +-
.../apache_beam/io/gcp/datastore/v1/helper.py | 18 +-
.../io/gcp/datastore/v1/query_splitter.py | 2 +-
sdks/python/apache_beam/io/gcp/gcsfilesystem.py | 243 ++
.../apache_beam/io/gcp/gcsfilesystem_test.py | 322 ++
sdks/python/apache_beam/io/gcp/gcsio.py | 33 +-
sdks/python/apache_beam/io/gcp/gcsio_test.py | 48 +-
.../io/gcp/tests/bigquery_matcher.py | 3 +-
sdks/python/apache_beam/io/iobase.py | 20 +-
sdks/python/apache_beam/io/localfilesystem.py | 235 ++
.../apache_beam/io/localfilesystem_test.py | 189 +
sdks/python/apache_beam/io/range_trackers.py | 19 +-
sdks/python/apache_beam/io/source_test_utils.py | 79 +-
.../apache_beam/io/source_test_utils_test.py | 20 +-
sdks/python/apache_beam/io/textio.py | 23 +-
sdks/python/apache_beam/io/textio_test.py | 20 +-
sdks/python/apache_beam/io/tfrecordio.py | 26 +-
sdks/python/apache_beam/io/tfrecordio_test.py | 38 +-
sdks/python/apache_beam/metrics/cells.py | 28 +-
sdks/python/apache_beam/metrics/execution.py | 3 +-
sdks/python/apache_beam/metrics/metric.py | 9 +-
sdks/python/apache_beam/pipeline.py | 138 +-
sdks/python/apache_beam/pipeline_test.py | 31 +-
sdks/python/apache_beam/pvalue.py | 317 +-
sdks/python/apache_beam/pvalue_test.py | 33 -
.../runners/api/beam_runner_api_pb2.py | 272 +-
sdks/python/apache_beam/runners/common.pxd | 2 +-
sdks/python/apache_beam/runners/common.py | 15 +-
.../runners/dataflow/dataflow_metrics.py | 86 +-
.../runners/dataflow/dataflow_metrics_test.py | 66 +-
.../runners/dataflow/dataflow_runner.py | 164 +-
.../runners/dataflow/dataflow_runner_test.py | 67 +-
.../runners/dataflow/internal/apiclient.py | 63 +-
.../runners/dataflow/internal/apiclient_test.py | 12 +-
.../runners/dataflow/internal/dependency.py | 30 +-
.../runners/dataflow/native_io/iobase_test.py | 173 +
.../runners/dataflow/test_dataflow_runner.py | 26 +-
.../runners/direct/bundle_factory.py | 17 +-
sdks/python/apache_beam/runners/direct/clock.py | 9 +-
.../consumer_tracking_pipeline_visitor.py | 11 +-
.../consumer_tracking_pipeline_visitor_test.py | 28 +-
.../apache_beam/runners/direct/direct_runner.py | 22 +-
.../runners/direct/evaluation_context.py | 77 +-
.../apache_beam/runners/direct/executor.py | 28 +-
.../runners/direct/transform_evaluator.py | 116 +-
.../runners/direct/transform_result.py | 45 +-
.../runners/direct/watermark_manager.py | 4 +-
sdks/python/apache_beam/runners/runner.py | 16 +-
.../apache_beam/tests/pipeline_verifiers.py | 46 +-
.../tests/pipeline_verifiers_test.py | 47 +-
sdks/python/apache_beam/transforms/combiners.py | 62 +-
.../apache_beam/transforms/combiners_test.py | 4 +-
sdks/python/apache_beam/transforms/core.py | 204 +-
.../apache_beam/transforms/create_test.py | 121 +
sdks/python/apache_beam/transforms/display.py | 4 +-
.../apache_beam/transforms/display_test.py | 36 +
.../python/apache_beam/transforms/ptransform.py | 42 +-
.../apache_beam/transforms/ptransform_test.py | 41 +-
.../python/apache_beam/transforms/sideinputs.py | 143 +-
.../apache_beam/transforms/sideinputs_test.py | 99 +-
sdks/python/apache_beam/transforms/trigger.py | 40 +-
.../apache_beam/transforms/trigger_test.py | 6 +-
sdks/python/apache_beam/transforms/util.py | 20 +-
sdks/python/apache_beam/transforms/util_test.py | 50 +
sdks/python/apache_beam/transforms/window.py | 9 +-
.../apache_beam/transforms/window_test.py | 8 +-
sdks/python/apache_beam/typehints/decorators.py | 26 +-
.../apache_beam/typehints/trivial_inference.py | 30 +-
.../typehints/trivial_inference_test.py | 3 +-
sdks/python/apache_beam/typehints/typecheck.py | 11 +-
sdks/python/apache_beam/typehints/typehints.py | 66 +-
.../apache_beam/typehints/typehints_test.py | 7 +-
.../apache_beam/utils/annotations_test.py | 2 +-
sdks/python/apache_beam/utils/counters.py | 5 +-
sdks/python/apache_beam/utils/path.py | 3 +-
.../apache_beam/utils/pipeline_options.py | 106 +-
.../apache_beam/utils/pipeline_options_test.py | 52 +-
sdks/python/apache_beam/utils/proto_utils.py | 17 +-
sdks/python/apache_beam/utils/retry.py | 20 +-
sdks/python/apache_beam/utils/test_stream.py | 163 +
.../apache_beam/utils/test_stream_test.py | 82 +
sdks/python/apache_beam/utils/timestamp.py | 6 +-
sdks/python/apache_beam/utils/urns.py | 2 +
sdks/python/apache_beam/utils/value_provider.py | 103 +
.../apache_beam/utils/value_provider_test.py | 145 +
sdks/python/apache_beam/utils/windowed_value.py | 17 +-
sdks/python/apache_beam/version.py | 36 -
sdks/python/findSupportedPython.groovy | 80 +
sdks/python/generate_pydoc.sh | 2 +-
sdks/python/pom.xml | 31 +-
sdks/python/run_postcommit.sh | 35 +-
sdks/python/run_pylint.sh | 27 +-
sdks/python/setup.py | 19 +-
sdks/python/tox.ini | 18 +-
1164 files changed, 69329 insertions(+), 54774 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/46c41fce/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
----------------------------------------------------------------------
diff --cc .test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
index 0000000,1348a19..e1cbafe
mode 000000,100644..100644
--- a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
+++ b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
@@@ -1,0 -1,49 +1,49 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ import common_job_properties
+
+ // This job runs the suite of ValidatesRunner tests against the Gearpump
+ // runner.
+ mavenJob('beam_PostCommit_Java_ValidatesRunner_Gearpump') {
+ description('Runs the ValidatesRunner suite on the Gearpump runner.')
+
+ previousNames('beam_PostCommit_Java_RunnableOnService_Gearpump')
+
+ // Set common parameters.
+ common_job_properties.setTopLevelMainJobProperties(
+ delegate,
+ 'gearpump-runner')
+
+ // Set maven parameters.
+ common_job_properties.setMavenConfig(delegate)
+
+ // Sets that this is a PostCommit job.
+ // 0 5 31 2 * will run on Feb 31 (i.e. never) according to job properties.
+ // In post-commit this job triggers only on SCM changes.
+ common_job_properties.setPostCommit(delegate, '0 5 31 2 *')
+
+ // Allows triggering this build against pull requests.
+ common_job_properties.enablePhraseTriggeringFromPullRequest(
+ delegate,
+ 'Apache Gearpump Runner ValidatesRunner Tests',
+ 'Run Gearpump ValidatesRunner')
+
+ // Maven goals for this job.
- goals('-B -e clean verify -am -pl runners/gearpump -DforkCount=0 -DvalidatesRunnerPipelineOptions=\'[ "--runner=TestGearpumpRunner", "--streaming=false" ]\'')
++ goals('-B -e clean verify -am -pl runners/gearpump -DforkCount=0 -DvalidatesRunnerPipelineOptions=\'[ "--runner=TestGearpumpRunner"]\'')
+ }
http://git-wip-us.apache.org/repos/asf/beam/blob/46c41fce/.travis.yml
----------------------------------------------------------------------
diff --cc .travis.yml
index 87ade2f,0b5d700..a1b28f9
--- a/.travis.yml
+++ b/.travis.yml
@@@ -78,11 -77,10 +77,11 @@@ install
- if [ "$TEST_PYTHON" ] && ! pip list | grep tox; then travis_retry pip install tox --user; fi
# Removing this here protects from inadvertent caching
- rm -rf "$HOME/.m2/repository/org/apache/beam"
+ - rm -rf "$HOME/.m2/repository/org/apache/gearpump"
script:
- - if [ "$TEST_PYTHON" ]; then travis_retry $TOX_HOME/tox -e $TOX_ENV -c sdks/python/tox.ini; fi
- - if [ ! "$TEST_PYTHON" ]; then travis_retry mvn --batch-mode --update-snapshots --no-snapshot-updates --threads 1C $MAVEN_OVERRIDE install && travis_retry bash -ex .travis/test_wordcount.sh; fi
+ - if [ "$TEST_PYTHON" ]; then travis_retry $TOX_HOME/tox -c sdks/python/tox.ini; fi
+ - if [ ! "$TEST_PYTHON" ]; then travis_retry mvn --batch-mode --update-snapshots --no-snapshot-updates --threads 1C $MAVEN_OVERRIDE install && travis_retry bash -ex .test-infra/travis/test_wordcount.sh; fi
cache:
directories:
http://git-wip-us.apache.org/repos/asf/beam/blob/46c41fce/examples/java/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/46c41fce/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/46c41fce/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoMultiOutputTranslator.java
----------------------------------------------------------------------
diff --cc runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoMultiOutputTranslator.java
index 0000000,0000000..e78568d
new file mode 100644
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoMultiOutputTranslator.java
@@@ -1,0 -1,0 +1,104 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.beam.runners.gearpump.translators;
++
++import java.util.ArrayList;
++import java.util.Collection;
++import java.util.List;
++import java.util.Map;
++
++import org.apache.beam.runners.gearpump.translators.functions.DoFnFunction;
++import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
++import org.apache.beam.sdk.transforms.DoFn;
++import org.apache.beam.sdk.transforms.ParDo;
++import org.apache.beam.sdk.util.WindowedValue;
++import org.apache.beam.sdk.values.PCollection;
++import org.apache.beam.sdk.values.PCollectionView;
++import org.apache.beam.sdk.values.TaggedPValue;
++import org.apache.beam.sdk.values.TupleTag;
++
++import org.apache.gearpump.streaming.dsl.api.functions.FilterFunction;
++import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
++
++/**
++ * {@link ParDo.MultiOutput} is translated to Gearpump flatMap function
++ * with {@link DoFn} wrapped in {@link DoFnFunction}. The outputs are
++ * further filtered with Gearpump filter function by output tag
++ */
++@SuppressWarnings({"rawtypes", "unchecked"})
++public class ParDoMultiOutputTranslator<InputT, OutputT> implements
++ TransformTranslator<ParDo.MultiOutput<InputT, OutputT>> {
++
++ private static final long serialVersionUID = -6023461558200028849L;
++
++ @Override
++ public void translate(ParDo.MultiOutput<InputT, OutputT> transform, TranslationContext context) {
++ PCollection<InputT> inputT = (PCollection<InputT>) context.getInput();
++ JavaStream<WindowedValue<InputT>> inputStream = context.getInputStream(inputT);
++ Collection<PCollectionView<?>> sideInputs = transform.getSideInputs();
++ Map<String, PCollectionView<?>> tagsToSideInputs =
++ TranslatorUtils.getTagsToSideInputs(sideInputs);
++
++ List<TaggedPValue> outputs = context.getOutputs();
++ final TupleTag<OutputT> mainOutput = transform.getMainOutputTag();
++ List<TupleTag<?>> sideOutputs = new ArrayList<>(outputs.size() - 1);
++ for (TaggedPValue output: outputs) {
++ TupleTag<?> tag = output.getTag();
++ if (tag != null && !tag.getId().equals(mainOutput.getId())) {
++ sideOutputs.add(tag);
++ }
++ }
++
++ JavaStream<TranslatorUtils.RawUnionValue> unionStream = TranslatorUtils.withSideInputStream(
++ context, inputStream, tagsToSideInputs);
++
++ JavaStream<TranslatorUtils.RawUnionValue> outputStream =
++ TranslatorUtils.toList(unionStream).flatMap(
++ new DoFnFunction<>(
++ context.getPipelineOptions(),
++ transform.getFn(),
++ inputT.getWindowingStrategy(),
++ sideInputs,
++ tagsToSideInputs,
++ mainOutput,
++ sideOutputs), transform.getName());
++ for (TaggedPValue output: outputs) {
++ JavaStream<WindowedValue<OutputT>> taggedStream = outputStream
++ .filter(new FilterByOutputTag(output.getTag().getId()),
++ "filter_by_output_tag")
++ .map(new TranslatorUtils.FromRawUnionValue<OutputT>(), "from_RawUnionValue");
++ context.setOutputStream(output.getValue(), taggedStream);
++ }
++ }
++
++ private static class FilterByOutputTag extends FilterFunction<TranslatorUtils.RawUnionValue> {
++
++ private static final long serialVersionUID = 7276155265895637526L;
++ private final String tag;
++
++ FilterByOutputTag(String tag) {
++ this.tag = tag;
++ }
++
++ @Override
++ public boolean filter(TranslatorUtils.RawUnionValue value) {
++ return value.getUnionTag().equals(tag);
++ }
++ }
++}
http://git-wip-us.apache.org/repos/asf/beam/blob/46c41fce/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoSingleOutputTranslator.java
----------------------------------------------------------------------
diff --cc runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoSingleOutputTranslator.java
index 0000000,0000000..86879b7
new file mode 100644
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoSingleOutputTranslator.java
@@@ -1,0 -1,0 +1,75 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.beam.runners.gearpump.translators;
++
++import java.util.Collection;
++import java.util.List;
++import java.util.Map;
++
++import org.apache.beam.runners.gearpump.translators.functions.DoFnFunction;
++import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
++import org.apache.beam.sdk.transforms.DoFn;
++import org.apache.beam.sdk.transforms.ParDo;
++import org.apache.beam.sdk.util.WindowedValue;
++import org.apache.beam.sdk.util.WindowingStrategy;
++import org.apache.beam.sdk.values.PCollection;
++
++import org.apache.beam.sdk.values.PCollectionView;
++import org.apache.beam.sdk.values.TupleTag;
++import org.apache.beam.sdk.values.TupleTagList;
++import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
++
++/**
++ * {@link ParDo.SingleOutput} is translated to Gearpump flatMap function
++ * with {@link DoFn} wrapped in {@link DoFnFunction}.
++ */
++public class ParDoSingleOutputTranslator<InputT, OutputT> implements
++ TransformTranslator<ParDo.SingleOutput<InputT, OutputT>> {
++
++ private static final long serialVersionUID = -3413205558160983784L;
++ private final TupleTag<OutputT> mainOutput = new TupleTag<>();
++ private final List<TupleTag<?>> sideOutputs = TupleTagList.empty().getAll();
++
++ @Override
++ public void translate(ParDo.SingleOutput<InputT, OutputT> transform, TranslationContext context) {
++ DoFn<InputT, OutputT> doFn = transform.getFn();
++ PCollection<OutputT> output = (PCollection<OutputT>) context.getOutput();
++ WindowingStrategy<?, ?> windowingStrategy = output.getWindowingStrategy();
++
++ Collection<PCollectionView<?>> sideInputs = transform.getSideInputs();
++ Map<String, PCollectionView<?>> tagsToSideInputs =
++ TranslatorUtils.getTagsToSideInputs(sideInputs);
++ JavaStream<WindowedValue<InputT>> inputStream = context.getInputStream(
++ context.getInput());
++ JavaStream<TranslatorUtils.RawUnionValue> unionStream =
++ TranslatorUtils.withSideInputStream(context,
++ inputStream, tagsToSideInputs);
++
++ DoFnFunction<InputT, OutputT> doFnFunction = new DoFnFunction<>(context.getPipelineOptions(),
++ doFn, windowingStrategy, sideInputs, tagsToSideInputs,
++ mainOutput, sideOutputs);
++
++ JavaStream<WindowedValue<OutputT>> outputStream =
++ TranslatorUtils.toList(unionStream)
++ .flatMap(doFnFunction, transform.getName())
++ .map(new TranslatorUtils.FromRawUnionValue<OutputT>(), "from_RawUnionValue");
++
++ context.setOutputStream(context.getOutput(), outputStream);
++ }
++}
http://git-wip-us.apache.org/repos/asf/beam/blob/46c41fce/runners/pom.xml
----------------------------------------------------------------------
diff --cc runners/pom.xml
index dd791ff,8f3cabd..a114382
--- a/runners/pom.xml
+++ b/runners/pom.xml
@@@ -54,55 -54,6 +54,15 @@@
</plugins>
</build>
</profile>
+ <profile>
+ <id>java8</id>
+ <activation>
+ <jdk>[1.8,)</jdk>
+ </activation>
+ <modules>
+ <module>gearpump</module>
+ </modules>
+ </profile>
-
- <!-- A profile that adds an integration test phase if and only if
- the runnableOnServicePipelineOptions maven property has been set.
- It should be set to a valid PipelineOptions JSON string. -->
- <profile>
- <id>runnable-on-service-tests</id>
- <activation>
- <property><name>runnableOnServicePipelineOptions</name></property>
- </activation>
- <build>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <executions>
- <execution>
- <id>runnable-on-service-tests</id>
- <phase>integration-test</phase>
- <goals>
- <goal>test</goal>
- </goals>
- <configuration>
- <groups>org.apache.beam.sdk.testing.RunnableOnService</groups>
- <parallel>all</parallel>
- <threadCount>4</threadCount>
- <dependenciesToScan>
- <dependency>org.apache.beam:beam-sdks-java-core</dependency>
- </dependenciesToScan>
- <systemPropertyVariables>
- <beamTestPipelineOptions>${runnableOnServicePipelineOptions}</beamTestPipelineOptions>
- </systemPropertyVariables>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </pluginManagement>
- </build>
- </profile>
</profiles>
<build>