You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2017/04/19 19:15:24 UTC
[50/50] [abbrv] beam git commit: This closes #2584
This closes #2584
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/aa07a1d4
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/aa07a1d4
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/aa07a1d4
Branch: refs/heads/DSL_SQL
Commit: aa07a1d41b71c810c4968bd35e5ca69dc56b8c30
Parents: ca87603 19ae877
Author: Dan Halperin <dh...@google.com>
Authored: Wed Apr 19 12:14:05 2017 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Wed Apr 19 12:14:05 2017 -0700
----------------------------------------------------------------------
.jenkins/common_job_properties.groovy | 261 ----
.../job_beam_PerformanceTests_Dataflow.groovy | 43 -
.jenkins/job_beam_PerformanceTests_JDBC.groovy | 60 -
.jenkins/job_beam_PerformanceTests_Spark.groovy | 44 -
...job_beam_PostCommit_Java_MavenInstall.groovy | 42 -
..._PostCommit_Java_ValidatesRunner_Apex.groovy | 48 -
...tCommit_Java_ValidatesRunner_Dataflow.groovy | 45 -
...PostCommit_Java_ValidatesRunner_Flink.groovy | 43 -
...tCommit_Java_ValidatesRunner_Gearpump.groovy | 49 -
...PostCommit_Java_ValidatesRunner_Spark.groovy | 44 -
.../job_beam_PostCommit_Python_Verify.groovy | 55 -
.../job_beam_PreCommit_Java_MavenInstall.groovy | 42 -
.../job_beam_PreCommit_Website_Stage.groovy | 80 -
.jenkins/job_beam_PreCommit_Website_Test.groovy | 65 -
.../job_beam_Release_NightlySnapshot.groovy | 45 -
.jenkins/job_seed.groovy | 53 -
.../jenkins/common_job_properties.groovy | 261 ++++
.../job_beam_PerformanceTests_Dataflow.groovy | 43 +
.../job_beam_PerformanceTests_JDBC.groovy | 60 +
.../job_beam_PerformanceTests_Spark.groovy | 44 +
...job_beam_PostCommit_Java_MavenInstall.groovy | 42 +
..._PostCommit_Java_ValidatesRunner_Apex.groovy | 48 +
...tCommit_Java_ValidatesRunner_Dataflow.groovy | 45 +
...PostCommit_Java_ValidatesRunner_Flink.groovy | 43 +
...tCommit_Java_ValidatesRunner_Gearpump.groovy | 49 +
...PostCommit_Java_ValidatesRunner_Spark.groovy | 44 +
.../job_beam_PostCommit_Python_Verify.groovy | 55 +
.../job_beam_PreCommit_Java_MavenInstall.groovy | 42 +
.../job_beam_PreCommit_Website_Stage.groovy | 80 +
.../job_beam_PreCommit_Website_Test.groovy | 65 +
.../job_beam_Release_NightlySnapshot.groovy | 45 +
.test-infra/jenkins/job_seed.groovy | 53 +
.../cassandra-service-for-local-dev.yaml | 28 +
.../cassandra-svc-statefulset.yaml | 114 ++
.../LargeITCluster/cassandra-svc-temp.yaml | 74 +
.../cassandra/LargeITCluster/data-load.sh | 122 ++
.../cassandra/LargeITCluster/show_health.sh | 47 +
.../cassandra/LargeITCluster/start-up.sh | 22 +
.../cassandra/LargeITCluster/teardown.sh | 25 +
.../cassandra-service-for-local-dev.yaml | 30 +
.../SmallITCluster/cassandra-svc-rc.yaml | 74 +
.../cassandra/SmallITCluster/data-load.sh | 86 ++
.../cassandra/SmallITCluster/show_health.sh | 47 +
.../cassandra/SmallITCluster/start-up.sh | 23 +
.../cassandra/SmallITCluster/teardown.sh | 22 +
.../kubernetes/cassandra/data-load-setup.sh | 29 +
.../elasticsearch-service-for-local-dev.yaml | 33 +
.../es-services-deployments.yaml | 258 ++++
.../LargeProductionCluster/start-up.sh | 22 +
.../LargeProductionCluster/teardown.sh | 21 +
.../elasticsearch-service-for-local-dev.yaml | 34 +
.../SmallITCluster/elasticsearch-svc-rc.yaml | 96 ++
.../elasticsearch/SmallITCluster/start-up.sh | 23 +
.../elasticsearch/SmallITCluster/teardown.sh | 21 +
.../kubernetes/elasticsearch/data-load-setup.sh | 26 +
.../kubernetes/elasticsearch/data-load.sh | 33 +
.../kubernetes/elasticsearch/es_test_data.py | 299 ++++
.../kubernetes/elasticsearch/show-health.sh | 33 +
.../postgres/postgres-service-for-local-dev.yml | 28 +
.test-infra/kubernetes/postgres/postgres.yml | 56 +
.test-infra/travis/README.md | 23 +
.test-infra/travis/settings.xml | 33 +
.test-infra/travis/test_wordcount.sh | 125 ++
.travis.yml | 4 +-
.travis/README.md | 23 -
.travis/settings.xml | 33 -
.travis/test_wordcount.sh | 125 --
examples/java/pom.xml | 5 +
examples/java8/pom.xml | 39 +
pom.xml | 13 +
runners/apex/pom.xml | 1 +
.../apache/beam/runners/apex/ApexRunner.java | 32 +-
.../beam/runners/apex/ApexYarnLauncher.java | 2 -
.../apex/translation/ParDoTranslator.java | 6 +-
.../operators/ApexGroupByKeyOperator.java | 226 +--
.../operators/ApexParDoOperator.java | 51 +-
.../apex/translation/utils/NoOpStepContext.java | 2 +-
.../utils/SerializablePipelineOptions.java | 13 +-
.../apex/translation/utils/ValuesSource.java | 2 +-
.../apex/examples/UnboundedTextSource.java | 2 +-
.../translation/GroupByKeyTranslatorTest.java | 2 +-
.../apex/translation/ParDoTranslatorTest.java | 18 +-
.../translation/utils/CollectionSource.java | 2 +-
runners/core-construction-java/pom.xml | 2 +-
.../beam/runners/core/construction/Coders.java | 26 +-
.../DeduplicatedFlattenFactory.java | 63 +-
.../EmptyFlattenAsCreateFactory.java | 20 +-
.../runners/core/construction/PCollections.java | 97 ++
.../core/construction/PTransformMatchers.java | 2 -
.../construction/PTransformReplacements.java | 69 +
.../runners/core/construction/PTransforms.java | 107 ++
.../core/construction/PrimitiveCreate.java | 13 +-
.../core/construction/SdkComponents.java | 49 +-
.../SingleInputOutputOverrideFactory.java | 9 +-
.../runners/core/construction/Triggers.java | 336 +++++
.../UnboundedReadFromBoundedSource.java | 7 +-
.../core/construction/UnconsumedReads.java | 72 +
.../UnsupportedOverrideFactory.java | 14 +-
.../core/construction/WindowingStrategies.java | 1 -
.../runners/core/construction/CodersTest.java | 144 +-
.../DeduplicatedFlattenFactoryTest.java | 18 +-
.../EmptyFlattenAsCreateFactoryTest.java | 36 +-
.../core/construction/PCollectionsTest.java | 188 +++
.../PTransformReplacementsTest.java | 131 ++
.../core/construction/PTransformsTest.java | 189 +++
.../core/construction/SdkComponentsTest.java | 85 +-
.../SingleInputOutputOverrideFactoryTest.java | 31 +-
.../runners/core/construction/TriggersTest.java | 111 ++
.../core/construction/UnconsumedReadsTest.java | 105 ++
.../UnsupportedOverrideFactoryTest.java | 11 +-
runners/core-java/pom.xml | 5 +
.../beam/runners/core/BaseExecutionContext.java | 13 +-
.../apache/beam/runners/core/DoFnAdapters.java | 16 +-
.../apache/beam/runners/core/DoFnRunners.java | 40 +-
.../beam/runners/core/ExecutionContext.java | 13 +-
.../GroupAlsoByWindowViaOutputBufferDoFn.java | 2 +-
.../core/GroupAlsoByWindowViaWindowSetDoFn.java | 2 +-
.../GroupAlsoByWindowViaWindowSetNewDoFn.java | 8 +-
.../org/apache/beam/runners/core/OldDoFn.java | 41 +-
...eBoundedSplittableProcessElementInvoker.java | 8 +-
.../beam/runners/core/OutputWindowedValue.java | 10 +-
.../beam/runners/core/ProcessFnRunner.java | 127 ++
.../core/PushbackSideInputDoFnRunner.java | 106 +-
.../beam/runners/core/SimpleDoFnRunner.java | 54 +-
.../beam/runners/core/SimpleOldDoFnRunner.java | 63 +-
.../core/SimplePushbackSideInputDoFnRunner.java | 115 ++
.../beam/runners/core/SplittableParDo.java | 131 +-
.../beam/runners/core/WindowingInternals.java | 8 +-
.../core/WindowingInternalsAdapters.java | 8 +-
.../core/GroupAlsoByWindowsProperties.java | 10 +-
.../apache/beam/runners/core/NoOpOldDoFn.java | 4 +-
.../apache/beam/runners/core/OldDoFnTest.java | 4 +-
...ndedSplittableProcessElementInvokerTest.java | 6 +-
.../core/PushbackSideInputDoFnRunnerTest.java | 282 ----
.../beam/runners/core/ReduceFnTester.java | 10 +-
.../runners/core/SimpleOldDoFnRunnerTest.java | 4 +-
.../SimplePushbackSideInputDoFnRunnerTest.java | 282 ++++
.../beam/runners/core/SplittableParDoTest.java | 98 +-
runners/direct-java/pom.xml | 1 +
.../direct/BoundedReadEvaluatorFactory.java | 2 +-
...ectGBKIntoKeyedWorkItemsOverrideFactory.java | 16 +-
.../direct/DirectGroupByKeyOverrideFactory.java | 14 +-
...ecycleManagerRemovingTransformEvaluator.java | 6 +-
.../GroupAlsoByWindowEvaluatorFactory.java | 10 +-
.../beam/runners/direct/ModelEnforcement.java | 13 +-
.../beam/runners/direct/ParDoEvaluator.java | 129 +-
.../runners/direct/ParDoEvaluatorFactory.java | 23 +-
.../direct/ParDoMultiOverrideFactory.java | 24 +-
...littableProcessElementsEvaluatorFactory.java | 108 +-
.../direct/StatefulParDoEvaluatorFactory.java | 6 +-
.../direct/TestStreamEvaluatorFactory.java | 14 +-
.../direct/TransformEvaluatorFactory.java | 10 +-
.../direct/TransformEvaluatorRegistry.java | 4 +-
.../direct/UnboundedReadEvaluatorFactory.java | 2 +-
.../runners/direct/ViewOverrideFactory.java | 18 +-
.../direct/WriteWithShardingFactory.java | 16 +-
.../direct/BoundedReadEvaluatorFactoryTest.java | 4 +-
.../DirectGroupByKeyOverrideFactoryTest.java | 12 +-
.../beam/runners/direct/DirectRunnerTest.java | 4 +-
...leManagerRemovingTransformEvaluatorTest.java | 8 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 13 +-
.../direct/ParDoMultiOverrideFactoryTest.java | 45 -
.../direct/TestStreamEvaluatorFactoryTest.java | 12 -
.../UnboundedReadEvaluatorFactoryTest.java | 2 +-
.../runners/direct/ViewOverrideFactoryTest.java | 42 +-
.../direct/WriteWithShardingFactoryTest.java | 23 +-
runners/flink/examples/pom.xml | 126 --
.../beam/runners/flink/examples/TFIDF.java | 455 ------
.../beam/runners/flink/examples/WordCount.java | 129 --
.../runners/flink/examples/package-info.java | 22 -
.../flink/examples/streaming/AutoComplete.java | 400 -----
.../flink/examples/streaming/JoinExamples.java | 154 --
.../examples/streaming/KafkaIOExamples.java | 338 -----
.../KafkaWindowedWordCountExample.java | 164 --
.../examples/streaming/WindowedWordCount.java | 141 --
.../flink/examples/streaming/package-info.java | 22 -
runners/flink/pom.xml | 320 +++-
runners/flink/runner/pom.xml | 311 ----
.../flink/DefaultParallelismFactory.java | 39 -
.../flink/FlinkBatchPipelineTranslator.java | 139 --
.../flink/FlinkBatchTransformTranslators.java | 723 ---------
.../flink/FlinkBatchTranslationContext.java | 153 --
.../flink/FlinkDetachedRunnerResult.java | 76 -
.../FlinkPipelineExecutionEnvironment.java | 241 ---
.../runners/flink/FlinkPipelineOptions.java | 101 --
.../runners/flink/FlinkPipelineTranslator.java | 53 -
.../apache/beam/runners/flink/FlinkRunner.java | 232 ---
.../runners/flink/FlinkRunnerRegistrar.java | 62 -
.../beam/runners/flink/FlinkRunnerResult.java | 98 --
.../flink/FlinkStreamingPipelineTranslator.java | 272 ----
.../FlinkStreamingTransformTranslators.java | 1107 --------------
.../flink/FlinkStreamingTranslationContext.java | 130 --
.../flink/FlinkStreamingViewOverrides.java | 372 -----
.../flink/PipelineTranslationOptimizer.java | 72 -
.../beam/runners/flink/TestFlinkRunner.java | 84 --
.../beam/runners/flink/TranslationMode.java | 31 -
.../apache/beam/runners/flink/package-info.java | 22 -
.../functions/FlinkAggregatorFactory.java | 53 -
.../functions/FlinkAssignContext.java | 63 -
.../functions/FlinkAssignWindows.java | 49 -
.../functions/FlinkDoFnFunction.java | 161 --
.../FlinkMergingNonShuffleReduceFunction.java | 228 ---
.../FlinkMergingPartialReduceFunction.java | 201 ---
.../functions/FlinkMergingReduceFunction.java | 199 ---
.../FlinkMultiOutputPruningFunction.java | 50 -
.../functions/FlinkNoOpStepContext.java | 73 -
.../functions/FlinkPartialReduceFunction.java | 172 ---
.../functions/FlinkReduceFunction.java | 173 ---
.../functions/FlinkSideInputReader.java | 80 -
.../functions/FlinkStatefulDoFnFunction.java | 198 ---
.../functions/SideInputInitializer.java | 73 -
.../translation/functions/package-info.java | 22 -
.../runners/flink/translation/package-info.java | 22 -
.../translation/types/CoderTypeInformation.java | 120 --
.../translation/types/CoderTypeSerializer.java | 132 --
.../types/EncodedValueComparator.java | 195 ---
.../types/EncodedValueSerializer.java | 113 --
.../types/EncodedValueTypeInformation.java | 107 --
.../flink/translation/types/FlinkCoder.java | 63 -
.../types/InspectableByteArrayOutputStream.java | 34 -
.../flink/translation/types/KvKeySelector.java | 50 -
.../flink/translation/types/package-info.java | 22 -
.../utils/SerializedPipelineOptions.java | 65 -
.../flink/translation/utils/package-info.java | 22 -
.../wrappers/DataInputViewWrapper.java | 58 -
.../wrappers/DataOutputViewWrapper.java | 51 -
.../SerializableFnAggregatorWrapper.java | 98 --
.../translation/wrappers/SourceInputFormat.java | 149 --
.../translation/wrappers/SourceInputSplit.java | 52 -
.../translation/wrappers/package-info.java | 22 -
.../wrappers/streaming/DoFnOperator.java | 772 ----------
.../streaming/KvToByteBufferKeySelector.java | 56 -
.../streaming/SingletonKeyedWorkItem.java | 58 -
.../streaming/SingletonKeyedWorkItemCoder.java | 128 --
.../streaming/SplittableDoFnOperator.java | 150 --
.../wrappers/streaming/WindowDoFnOperator.java | 120 --
.../wrappers/streaming/WorkItemKeySelector.java | 56 -
.../streaming/io/BoundedSourceWrapper.java | 218 ---
.../streaming/io/UnboundedFlinkSink.java | 200 ---
.../streaming/io/UnboundedFlinkSource.java | 120 --
.../streaming/io/UnboundedSocketSource.java | 249 ---
.../streaming/io/UnboundedSourceWrapper.java | 476 ------
.../wrappers/streaming/io/package-info.java | 22 -
.../wrappers/streaming/package-info.java | 22 -
.../state/FlinkBroadcastStateInternals.java | 865 -----------
.../state/FlinkKeyGroupStateInternals.java | 487 ------
.../state/FlinkSplitStateInternals.java | 260 ----
.../streaming/state/FlinkStateInternals.java | 1053 -------------
.../state/KeyGroupCheckpointedOperator.java | 35 -
.../state/KeyGroupRestoringOperator.java | 32 -
.../wrappers/streaming/state/package-info.java | 22 -
.../runner/src/main/resources/log4j.properties | 23 -
.../flink/EncodedValueComparatorTest.java | 70 -
.../runners/flink/FlinkRunnerRegistrarTest.java | 48 -
.../beam/runners/flink/FlinkTestPipeline.java | 72 -
.../beam/runners/flink/PipelineOptionsTest.java | 184 ---
.../beam/runners/flink/ReadSourceITCase.java | 85 --
.../flink/ReadSourceStreamingITCase.java | 74 -
.../beam/runners/flink/WriteSinkITCase.java | 192 ---
.../flink/streaming/DoFnOperatorTest.java | 600 --------
.../FlinkBroadcastStateInternalsTest.java | 245 ---
.../FlinkKeyGroupStateInternalsTest.java | 262 ----
.../streaming/FlinkSplitStateInternalsTest.java | 101 --
.../streaming/FlinkStateInternalsTest.java | 395 -----
.../flink/streaming/GroupByNullKeyTest.java | 124 --
.../flink/streaming/TestCountingSource.java | 254 ----
.../streaming/TopWikipediaSessionsITCase.java | 133 --
.../streaming/UnboundedSourceWrapperTest.java | 464 ------
.../runners/flink/streaming/package-info.java | 22 -
.../src/test/resources/log4j-test.properties | 27 -
.../flink/DefaultParallelismFactory.java | 39 +
.../flink/FlinkBatchPipelineTranslator.java | 139 ++
.../flink/FlinkBatchTransformTranslators.java | 723 +++++++++
.../flink/FlinkBatchTranslationContext.java | 153 ++
.../flink/FlinkDetachedRunnerResult.java | 75 +
.../FlinkPipelineExecutionEnvironment.java | 241 +++
.../runners/flink/FlinkPipelineOptions.java | 101 ++
.../runners/flink/FlinkPipelineTranslator.java | 53 +
.../apache/beam/runners/flink/FlinkRunner.java | 232 +++
.../runners/flink/FlinkRunnerRegistrar.java | 62 +
.../beam/runners/flink/FlinkRunnerResult.java | 98 ++
.../flink/FlinkStreamingPipelineTranslator.java | 276 ++++
.../FlinkStreamingTransformTranslators.java | 1044 +++++++++++++
.../flink/FlinkStreamingTranslationContext.java | 130 ++
.../flink/FlinkStreamingViewOverrides.java | 372 +++++
.../flink/PipelineTranslationOptimizer.java | 72 +
.../beam/runners/flink/TestFlinkRunner.java | 84 ++
.../beam/runners/flink/TranslationMode.java | 31 +
.../apache/beam/runners/flink/package-info.java | 22 +
.../functions/FlinkAggregatorFactory.java | 53 +
.../functions/FlinkAssignContext.java | 63 +
.../functions/FlinkAssignWindows.java | 49 +
.../functions/FlinkDoFnFunction.java | 161 ++
.../FlinkMergingNonShuffleReduceFunction.java | 228 +++
.../FlinkMergingPartialReduceFunction.java | 201 +++
.../functions/FlinkMergingReduceFunction.java | 199 +++
.../FlinkMultiOutputPruningFunction.java | 50 +
.../functions/FlinkNoOpStepContext.java | 73 +
.../functions/FlinkPartialReduceFunction.java | 172 +++
.../functions/FlinkReduceFunction.java | 173 +++
.../functions/FlinkSideInputReader.java | 80 +
.../functions/FlinkStatefulDoFnFunction.java | 198 +++
.../functions/SideInputInitializer.java | 73 +
.../translation/functions/package-info.java | 22 +
.../runners/flink/translation/package-info.java | 22 +
.../translation/types/CoderTypeInformation.java | 120 ++
.../translation/types/CoderTypeSerializer.java | 132 ++
.../types/EncodedValueComparator.java | 195 +++
.../types/EncodedValueSerializer.java | 113 ++
.../types/EncodedValueTypeInformation.java | 98 ++
.../types/InspectableByteArrayOutputStream.java | 34 +
.../flink/translation/types/KvKeySelector.java | 50 +
.../flink/translation/types/package-info.java | 22 +
.../utils/SerializedPipelineOptions.java | 67 +
.../flink/translation/utils/package-info.java | 22 +
.../wrappers/DataInputViewWrapper.java | 58 +
.../wrappers/DataOutputViewWrapper.java | 51 +
.../SerializableFnAggregatorWrapper.java | 98 ++
.../translation/wrappers/SourceInputFormat.java | 150 ++
.../translation/wrappers/SourceInputSplit.java | 52 +
.../translation/wrappers/package-info.java | 22 +
.../wrappers/streaming/DoFnOperator.java | 774 ++++++++++
.../streaming/KvToByteBufferKeySelector.java | 56 +
.../streaming/SingletonKeyedWorkItem.java | 56 +
.../streaming/SingletonKeyedWorkItemCoder.java | 126 ++
.../streaming/SplittableDoFnOperator.java | 150 ++
.../wrappers/streaming/WindowDoFnOperator.java | 117 ++
.../wrappers/streaming/WorkItemKeySelector.java | 56 +
.../streaming/io/BoundedSourceWrapper.java | 218 +++
.../streaming/io/UnboundedSocketSource.java | 249 +++
.../streaming/io/UnboundedSourceWrapper.java | 476 ++++++
.../wrappers/streaming/io/package-info.java | 22 +
.../wrappers/streaming/package-info.java | 22 +
.../state/FlinkBroadcastStateInternals.java | 865 +++++++++++
.../state/FlinkKeyGroupStateInternals.java | 487 ++++++
.../state/FlinkSplitStateInternals.java | 260 ++++
.../streaming/state/FlinkStateInternals.java | 1053 +++++++++++++
.../state/KeyGroupCheckpointedOperator.java | 35 +
.../state/KeyGroupRestoringOperator.java | 32 +
.../wrappers/streaming/state/package-info.java | 22 +
.../flink/src/main/resources/log4j.properties | 23 +
.../flink/EncodedValueComparatorTest.java | 70 +
.../runners/flink/FlinkRunnerRegistrarTest.java | 48 +
.../beam/runners/flink/FlinkTestPipeline.java | 72 +
.../beam/runners/flink/PipelineOptionsTest.java | 184 +++
.../beam/runners/flink/ReadSourceITCase.java | 85 ++
.../flink/ReadSourceStreamingITCase.java | 74 +
.../beam/runners/flink/WriteSinkITCase.java | 192 +++
.../flink/streaming/DoFnOperatorTest.java | 600 ++++++++
.../FlinkBroadcastStateInternalsTest.java | 245 +++
.../FlinkKeyGroupStateInternalsTest.java | 262 ++++
.../streaming/FlinkSplitStateInternalsTest.java | 101 ++
.../streaming/FlinkStateInternalsTest.java | 395 +++++
.../flink/streaming/GroupByNullKeyTest.java | 124 ++
.../flink/streaming/TestCountingSource.java | 254 ++++
.../streaming/TopWikipediaSessionsITCase.java | 133 ++
.../streaming/UnboundedSourceWrapperTest.java | 464 ++++++
.../runners/flink/streaming/package-info.java | 22 +
.../src/test/resources/log4j-test.properties | 27 +
runners/google-cloud-dataflow-java/pom.xml | 62 +-
.../dataflow/BatchStatefulParDoOverrides.java | 44 +-
.../runners/dataflow/BatchViewOverrides.java | 23 +-
.../beam/runners/dataflow/DataflowRunner.java | 96 +-
.../dataflow/PrimitiveParDoSingleFactory.java | 15 +-
.../dataflow/ReshuffleOverrideFactory.java | 12 +-
.../dataflow/StreamingViewOverrides.java | 14 +-
.../dataflow/internal/CustomSources.java | 2 +-
.../dataflow/BatchViewOverridesTest.java | 4 +-
.../dataflow/DataflowPipelineJobTest.java | 2 +-
.../runners/dataflow/DataflowRunnerTest.java | 24 +
.../PrimitiveParDoSingleFactoryTest.java | 59 +-
runners/pom.xml | 40 -
runners/spark/pom.xml | 2 +-
.../beam/runners/spark/TestSparkRunner.java | 14 +-
.../beam/runners/spark/io/MicrobatchSource.java | 7 +-
.../beam/runners/spark/io/SourceDStream.java | 2 +-
.../apache/beam/runners/spark/io/SourceRDD.java | 4 +-
.../SparkGroupAlsoByWindowViaWindowSet.java | 13 +-
.../spark/translation/MultiDoFnFunction.java | 4 +-
...SparkGroupAlsoByWindowViaOutputBufferFn.java | 10 +-
.../spark/translation/SparkProcessContext.java | 2 +-
.../spark/translation/SparkRuntimeContext.java | 2 +
.../streaming/StreamingTransformTranslator.java | 2 +-
.../spark/util/SparkSideInputReader.java | 3 +-
sdks/common/fn-api/pom.xml | 5 -
.../fn-api/src/main/proto/beam_fn_api.proto | 174 +--
.../src/main/proto/beam_runner_api.proto | 4 +-
.../src/main/resources/beam/findbugs-filter.xml | 7 -
sdks/java/core/pom.xml | 49 +-
.../main/java/org/apache/beam/sdk/Pipeline.java | 58 +-
.../java/org/apache/beam/sdk/io/AvroIO.java | 13 +-
.../sdk/io/BoundedReadFromUnboundedSource.java | 4 +-
.../org/apache/beam/sdk/io/BoundedSource.java | 13 +-
.../apache/beam/sdk/io/CompressedSource.java | 4 +-
.../org/apache/beam/sdk/io/CountingSource.java | 2 +-
.../org/apache/beam/sdk/io/FileBasedSource.java | 9 +-
.../apache/beam/sdk/io/OffsetBasedSource.java | 4 +-
.../main/java/org/apache/beam/sdk/io/Sink.java | 2 +-
.../java/org/apache/beam/sdk/io/TFRecordIO.java | 639 +++-----
.../org/apache/beam/sdk/io/UnboundedSource.java | 15 +-
.../main/java/org/apache/beam/sdk/io/XmlIO.java | 477 ++++++
.../java/org/apache/beam/sdk/io/XmlSink.java | 226 +--
.../java/org/apache/beam/sdk/io/XmlSource.java | 191 +--
.../beam/sdk/options/BigQueryOptions.java | 32 -
.../options/CloudResourceManagerOptions.java | 40 -
.../DefaultPipelineOptionsRegistrar.java | 5 -
.../org/apache/beam/sdk/options/GcpOptions.java | 227 ---
.../org/apache/beam/sdk/options/GcsOptions.java | 158 --
.../beam/sdk/options/GoogleApiDebugOptions.java | 87 --
.../beam/sdk/options/PipelineOptions.java | 16 +-
.../apache/beam/sdk/options/PubsubOptions.java | 36 -
.../sdk/runners/PTransformOverrideFactory.java | 33 +-
.../apache/beam/sdk/runners/PipelineRunner.java | 7 +-
.../beam/sdk/testing/BigqueryMatcher.java | 256 ----
.../beam/sdk/testing/SourceTestUtils.java | 6 +-
.../apache/beam/sdk/testing/TestPipeline.java | 5 +-
...esSplittableParDoWithWindowedSideInputs.java | 26 +
.../beam/sdk/transforms/AppliedPTransform.java | 5 +
.../org/apache/beam/sdk/transforms/Combine.java | 8 +-
.../org/apache/beam/sdk/transforms/Create.java | 7 +-
.../org/apache/beam/sdk/transforms/DoFn.java | 14 +-
.../apache/beam/sdk/transforms/DoFnTester.java | 41 +-
.../beam/sdk/transforms/FlatMapElements.java | 4 +-
.../apache/beam/sdk/transforms/MapElements.java | 4 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 86 +-
.../apache/beam/sdk/transforms/Partition.java | 2 +-
.../sdk/transforms/display/DisplayData.java | 6 +
.../beam/sdk/transforms/windowing/Triggers.java | 322 ----
.../beam/sdk/transforms/windowing/Window.java | 1 +
.../beam/sdk/transforms/windowing/WindowFn.java | 13 -
.../beam/sdk/util/AppEngineEnvironment.java | 62 -
...AttemptAndTimeBoundedExponentialBackOff.java | 170 ---
.../util/AttemptBoundedExponentialBackOff.java | 86 --
.../apache/beam/sdk/util/CredentialFactory.java | 29 -
.../org/apache/beam/sdk/util/DefaultBucket.java | 105 --
.../beam/sdk/util/GcpCredentialFactory.java | 67 -
.../apache/beam/sdk/util/GcpProjectUtil.java | 106 --
.../beam/sdk/util/GcsIOChannelFactory.java | 111 --
.../sdk/util/GcsIOChannelFactoryRegistrar.java | 38 -
.../apache/beam/sdk/util/GcsPathValidator.java | 95 --
.../java/org/apache/beam/sdk/util/GcsUtil.java | 798 ----------
.../apache/beam/sdk/util/IOChannelUtils.java | 22 +-
.../util/IntervalBoundedExponentialBackOff.java | 89 --
.../beam/sdk/util/NoopCredentialFactory.java | 68 -
.../sdk/util/NullCredentialInitializer.java | 62 -
.../org/apache/beam/sdk/util/Transport.java | 178 ---
.../beam/sdk/values/PCollectionTuple.java | 3 +-
.../org/apache/beam/sdk/values/TupleTag.java | 26 +-
.../apache/beam/sdk/values/TupleTagList.java | 2 +-
.../apache/beam/sdk/values/TypeDescriptors.java | 25 +-
.../org/apache/beam/sdk/values/TypedPValue.java | 4 +-
.../org/apache/beam/SdkCoreApiSurfaceTest.java | 2 -
.../java/org/apache/beam/sdk/PipelineTest.java | 92 +-
.../org/apache/beam/sdk/io/AvroSourceTest.java | 10 +-
.../apache/beam/sdk/io/CountingSourceTest.java | 6 +-
.../apache/beam/sdk/io/FileBasedSourceTest.java | 8 +-
.../beam/sdk/io/OffsetBasedSourceTest.java | 8 +-
.../java/org/apache/beam/sdk/io/ReadTest.java | 4 +-
.../org/apache/beam/sdk/io/TFRecordIOTest.java | 20 +-
.../java/org/apache/beam/sdk/io/TextIOTest.java | 113 +-
.../org/apache/beam/sdk/io/XmlSinkTest.java | 89 +-
.../org/apache/beam/sdk/io/XmlSourceTest.java | 250 +--
.../apache/beam/sdk/metrics/MetricsTest.java | 2 +-
.../apache/beam/sdk/options/GcpOptionsTest.java | 171 ---
.../sdk/options/GoogleApiDebugOptionsTest.java | 145 --
.../sdk/options/PipelineOptionsFactoryTest.java | 4 +-
.../beam/sdk/runners/PipelineRunnerTest.java | 46 +-
.../runners/dataflow/TestCountingSource.java | 2 +-
.../beam/sdk/testing/BigqueryMatcherTest.java | 176 ---
.../beam/sdk/testing/SourceTestUtilsTest.java | 2 +-
.../beam/sdk/testing/StaticWindowsTest.java | 10 +-
.../beam/sdk/testing/TestPipelineTest.java | 6 +-
.../apache/beam/sdk/transforms/CreateTest.java | 20 +-
.../apache/beam/sdk/transforms/ParDoTest.java | 293 ++--
.../beam/sdk/transforms/SplittableDoFnTest.java | 107 +-
.../sdk/transforms/display/DisplayDataTest.java | 17 +
.../sdk/transforms/windowing/TriggersTest.java | 100 --
...mptAndTimeBoundedExponentialBackOffTest.java | 213 ---
.../AttemptBoundedExponentialBackOffTest.java | 85 --
.../apache/beam/sdk/util/DefaultBucketTest.java | 112 --
.../beam/sdk/util/GcpProjectUtilTest.java | 76 -
.../util/GcsIOChannelFactoryRegistrarTest.java | 44 -
.../beam/sdk/util/GcsIOChannelFactoryTest.java | 43 -
.../beam/sdk/util/GcsPathValidatorTest.java | 87 --
.../org/apache/beam/sdk/util/GcsUtilTest.java | 798 ----------
.../IntervalBoundedExponentialBackOffTest.java | 100 --
.../util/RetryHttpRequestInitializerTest.java | 290 ----
.../beam/sdk/values/PCollectionTupleTest.java | 8 +-
.../apache/beam/sdk/values/TypedPValueTest.java | 46 +-
sdks/java/extensions/gcp-core/pom.xml | 217 +++
.../beam/sdk/options/BigQueryOptions.java | 32 +
.../options/CloudResourceManagerOptions.java | 40 +
.../org/apache/beam/sdk/options/GcpOptions.java | 227 +++
.../options/GcpPipelineOptionsRegistrar.java | 39 +
.../org/apache/beam/sdk/options/GcsOptions.java | 154 ++
.../beam/sdk/options/GoogleApiDebugOptions.java | 87 ++
.../apache/beam/sdk/options/PubsubOptions.java | 36 +
.../apache/beam/sdk/options/package-info.java | 22 +
.../beam/sdk/testing/BigqueryMatcher.java | 256 ++++
.../apache/beam/sdk/testing/package-info.java | 21 +
.../apache/beam/sdk/util/CredentialFactory.java | 29 +
.../org/apache/beam/sdk/util/DefaultBucket.java | 105 ++
.../beam/sdk/util/GcpCredentialFactory.java | 67 +
.../apache/beam/sdk/util/GcpProjectUtil.java | 106 ++
.../beam/sdk/util/GcsIOChannelFactory.java | 111 ++
.../sdk/util/GcsIOChannelFactoryRegistrar.java | 38 +
.../apache/beam/sdk/util/GcsPathValidator.java | 95 ++
.../java/org/apache/beam/sdk/util/GcsUtil.java | 798 ++++++++++
.../beam/sdk/util/NoopCredentialFactory.java | 68 +
.../sdk/util/NullCredentialInitializer.java | 62 +
.../org/apache/beam/sdk/util/Transport.java | 178 +++
.../org/apache/beam/sdk/util/package-info.java | 20 +
.../org/apache/beam/GcpCoreApiSurfaceTest.java | 62 +
.../apache/beam/sdk/options/GcpOptionsTest.java | 171 +++
.../sdk/options/GoogleApiDebugOptionsTest.java | 145 ++
.../beam/sdk/testing/BigqueryMatcherTest.java | 176 +++
.../apache/beam/sdk/util/DefaultBucketTest.java | 112 ++
.../beam/sdk/util/GcpProjectUtilTest.java | 76 +
.../util/GcsIOChannelFactoryRegistrarTest.java | 44 +
.../beam/sdk/util/GcsIOChannelFactoryTest.java | 43 +
.../beam/sdk/util/GcsPathValidatorTest.java | 87 ++
.../org/apache/beam/sdk/util/GcsUtilTest.java | 798 ++++++++++
.../util/RetryHttpRequestInitializerTest.java | 290 ++++
sdks/java/extensions/pom.xml | 1 +
sdks/java/harness/pom.xml | 5 +
.../beam/fn/harness/fake/FakeStepContext.java | 2 +-
.../control/ProcessBundleHandlerTest.java | 30 +-
.../sdk/io/elasticsearch/ElasticsearchIO.java | 2 +-
.../sdk/io/elasticsearch/ElasticsearchIOIT.java | 2 +-
.../io/elasticsearch/ElasticsearchIOTest.java | 4 +-
sdks/java/io/google-cloud-platform/pom.xml | 20 +-
.../sdk/io/gcp/bigquery/BatchLoadBigQuery.java | 180 ---
.../beam/sdk/io/gcp/bigquery/BatchLoads.java | 225 +++
.../sdk/io/gcp/bigquery/BigQueryHelpers.java | 13 +
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 172 +--
.../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 33 +-
.../io/gcp/bigquery/BigQueryTableSource.java | 34 +-
.../beam/sdk/io/gcp/bigquery/CreateTables.java | 127 ++
.../io/gcp/bigquery/GenerateShardedTable.java | 47 +
.../beam/sdk/io/gcp/bigquery/PrepareWrite.java | 81 +
.../beam/sdk/io/gcp/bigquery/ShardedKey.java | 25 +-
.../sdk/io/gcp/bigquery/StreamWithDeDup.java | 90 --
.../sdk/io/gcp/bigquery/StreamingInserts.java | 79 +
.../sdk/io/gcp/bigquery/StreamingWriteFn.java | 81 +-
.../io/gcp/bigquery/StreamingWriteTables.java | 86 ++
.../sdk/io/gcp/bigquery/TableDestination.java | 76 +
.../io/gcp/bigquery/TableDestinationCoder.java | 60 +
.../sdk/io/gcp/bigquery/TableRowWriter.java | 19 +-
.../sdk/io/gcp/bigquery/TagWithUniqueIds.java | 62 +
.../gcp/bigquery/TagWithUniqueIdsAndTable.java | 135 --
.../sdk/io/gcp/bigquery/TransformingSource.java | 4 +-
.../beam/sdk/io/gcp/bigquery/WriteBundles.java | 82 -
.../io/gcp/bigquery/WriteBundlesToFiles.java | 157 ++
.../sdk/io/gcp/bigquery/WritePartition.java | 163 +-
.../beam/sdk/io/gcp/bigquery/WriteRename.java | 71 +-
.../beam/sdk/io/gcp/bigquery/WriteTables.java | 58 +-
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 6 +-
.../io/gcp/pubsub/PubsubUnboundedSource.java | 6 +-
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 1419 +++++++-----------
.../io/gcp/bigquery/FakeBigQueryServices.java | 166 ++
.../sdk/io/gcp/bigquery/FakeDatasetService.java | 208 +++
.../sdk/io/gcp/bigquery/FakeJobService.java | 404 +++++
.../sdk/io/gcp/bigquery/TableContainer.java | 61 +
.../sdk/io/gcp/bigtable/BigtableIOTest.java | 8 +-
.../gcp/pubsub/PubsubUnboundedSourceTest.java | 4 +-
.../hadoop/inputformat/HadoopInputFormatIO.java | 4 +-
.../inputformat/HadoopInputFormatIOTest.java | 19 +-
.../integration/tests/HIFIOCassandraIT.java | 6 +-
.../integration/tests/HIFIOElasticIT.java | 4 +-
.../SmallITCluster/cassandra-svc-rc.yaml | 88 --
.../cassandra/SmallITCluster/start-up.sh | 21 -
.../cassandra/SmallITCluster/teardown.sh | 21 -
.../kubernetes/cassandra/data-load-setup.sh | 29 -
.../resources/kubernetes/cassandra/data-load.sh | 67 -
.../LargeProductionCluster/es-services.yaml | 277 ----
.../LargeProductionCluster/start-up.sh | 21 -
.../LargeProductionCluster/teardown.sh | 20 -
.../SmallITCluster/elasticsearch-svc-rc.yaml | 84 --
.../elasticsearch/SmallITCluster/start-up.sh | 22 -
.../elasticsearch/SmallITCluster/teardown.sh | 20 -
.../kubernetes/elasticsearch/data-load-setup.sh | 26 -
.../kubernetes/elasticsearch/data-load.sh | 33 -
.../kubernetes/elasticsearch/es_test_data.py | 299 ----
.../kubernetes/elasticsearch/show-health.sh | 25 -
sdks/java/io/hadoop/pom.xml | 15 +-
.../org/apache/beam/sdk/io/hbase/HBaseIO.java | 7 +-
.../apache/beam/sdk/io/hbase/HBaseIOTest.java | 2 +-
.../apache/beam/sdk/io/hdfs/HDFSFileSource.java | 6 +-
.../beam/sdk/io/hdfs/HDFSFileSourceTest.java | 4 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 42 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 10 +-
.../postgres-service-for-local-dev.yml | 28 -
.../src/test/resources/kubernetes/postgres.yml | 56 -
.../jdbc/src/test/resources/kubernetes/setup.sh | 19 -
.../src/test/resources/kubernetes/teardown.sh | 19 -
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 2 +-
.../org/apache/beam/sdk/io/jms/JmsIOTest.java | 4 +-
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 12 +-
.../apache/beam/sdk/io/kafka/KafkaIOTest.java | 8 +-
.../beam/sdk/io/kinesis/KinesisSource.java | 2 +-
.../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 11 +-
.../apache/beam/sdk/io/mongodb/MongoDbIO.java | 2 +-
.../sdk/io/mongodb/MongoDBGridFSIOTest.java | 2 +-
.../org/apache/beam/sdk/io/mqtt/MqttIO.java | 2 +-
sdks/python/.pylintrc | 6 +-
sdks/python/apache_beam/coders/coder_impl.py | 50 +-
sdks/python/apache_beam/coders/coders.py | 6 +-
sdks/python/apache_beam/error.py | 4 +
.../examples/complete/game/user_score.py | 8 +-
.../examples/complete/top_wikipedia_sessions.py | 8 -
.../examples/cookbook/datastore_wordcount.py | 15 +-
.../examples/cookbook/group_with_coder.py | 6 +-
.../examples/cookbook/multiple_output_pardo.py | 47 +-
.../examples/snippets/snippets_test.py | 33 +-
sdks/python/apache_beam/examples/wordcount.py | 32 +-
.../apache_beam/examples/wordcount_debugging.py | 4 -
sdks/python/apache_beam/internal/gcp/auth.py | 80 +-
.../apache_beam/internal/gcp/auth_test.py | 44 -
.../apache_beam/internal/gcp/json_value.py | 6 +
sdks/python/apache_beam/internal/pickler.py | 8 +-
sdks/python/apache_beam/io/avroio_test.py | 14 +-
sdks/python/apache_beam/io/concat_source.py | 74 +-
.../python/apache_beam/io/concat_source_test.py | 12 +-
sdks/python/apache_beam/io/filebasedsource.py | 53 +-
.../apache_beam/io/filebasedsource_test.py | 25 +-
sdks/python/apache_beam/io/fileio.py | 158 +-
sdks/python/apache_beam/io/fileio_test.py | 45 +-
sdks/python/apache_beam/io/filesystem.py | 1 +
sdks/python/apache_beam/io/filesystems_util.py | 10 +-
sdks/python/apache_beam/io/gcp/bigquery.py | 32 +-
.../io/gcp/datastore/v1/datastoreio.py | 24 +-
.../io/gcp/datastore/v1/datastoreio_test.py | 4 +-
.../apache_beam/io/gcp/datastore/v1/helper.py | 16 +-
.../io/gcp/datastore/v1/query_splitter.py | 2 +-
sdks/python/apache_beam/io/gcp/gcsfilesystem.py | 7 +-
.../io/gcp/tests/bigquery_matcher.py | 3 +-
sdks/python/apache_beam/io/iobase.py | 7 +-
sdks/python/apache_beam/io/localfilesystem.py | 3 +-
sdks/python/apache_beam/io/range_trackers.py | 19 +-
sdks/python/apache_beam/io/source_test_utils.py | 79 +-
.../apache_beam/io/source_test_utils_test.py | 20 +-
sdks/python/apache_beam/io/textio.py | 15 +-
sdks/python/apache_beam/io/textio_test.py | 18 +-
sdks/python/apache_beam/metrics/cells.py | 28 +-
sdks/python/apache_beam/metrics/execution.py | 3 +-
sdks/python/apache_beam/metrics/metric.py | 9 +-
sdks/python/apache_beam/pvalue.py | 26 +-
sdks/python/apache_beam/runners/common.pxd | 2 +-
sdks/python/apache_beam/runners/common.py | 15 +-
.../runners/dataflow/dataflow_metrics_test.py | 3 +-
.../runners/dataflow/dataflow_runner.py | 29 +-
.../runners/dataflow/internal/apiclient.py | 9 +-
.../runners/dataflow/internal/dependency.py | 6 +-
.../runners/dataflow/test_dataflow_runner.py | 4 -
.../runners/direct/bundle_factory.py | 14 +-
sdks/python/apache_beam/runners/direct/clock.py | 9 +-
.../consumer_tracking_pipeline_visitor_test.py | 2 +-
.../apache_beam/runners/direct/direct_runner.py | 6 +
.../runners/direct/evaluation_context.py | 21 +-
.../apache_beam/runners/direct/executor.py | 21 +-
.../runners/direct/transform_evaluator.py | 19 +-
.../runners/direct/transform_result.py | 45 +-
.../runners/direct/watermark_manager.py | 4 +-
sdks/python/apache_beam/runners/runner.py | 17 +-
.../apache_beam/tests/pipeline_verifiers.py | 7 +-
sdks/python/apache_beam/transforms/combiners.py | 62 +-
.../apache_beam/transforms/combiners_test.py | 4 +-
sdks/python/apache_beam/transforms/core.py | 112 +-
.../apache_beam/transforms/create_test.py | 18 +-
sdks/python/apache_beam/transforms/display.py | 4 +-
.../apache_beam/transforms/display_test.py | 36 +
.../python/apache_beam/transforms/ptransform.py | 38 +-
.../apache_beam/transforms/ptransform_test.py | 19 +-
.../python/apache_beam/transforms/sideinputs.py | 11 +-
sdks/python/apache_beam/transforms/trigger.py | 26 +-
.../apache_beam/transforms/trigger_test.py | 6 +-
sdks/python/apache_beam/typehints/decorators.py | 26 +-
.../apache_beam/typehints/trivial_inference.py | 26 +-
.../typehints/trivial_inference_test.py | 3 +-
sdks/python/apache_beam/typehints/typecheck.py | 11 +-
sdks/python/apache_beam/typehints/typehints.py | 66 +-
.../apache_beam/typehints/typehints_test.py | 7 +-
.../apache_beam/utils/annotations_test.py | 2 +-
sdks/python/apache_beam/utils/path.py | 3 +-
.../apache_beam/utils/pipeline_options.py | 95 +-
.../apache_beam/utils/pipeline_options_test.py | 52 +-
sdks/python/apache_beam/utils/proto_utils.py | 15 +-
sdks/python/apache_beam/utils/retry.py | 14 +-
sdks/python/apache_beam/utils/timestamp.py | 6 +-
sdks/python/apache_beam/utils/value_provider.py | 103 ++
.../apache_beam/utils/value_provider_test.py | 145 ++
sdks/python/apache_beam/utils/windowed_value.py | 17 +-
sdks/python/run_pylint.sh | 2 +-
sdks/python/tox.ini | 18 +-
694 files changed, 32745 insertions(+), 32281 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/aa07a1d4/pom.xml
----------------------------------------------------------------------