You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by jb...@apache.org on 2017/07/20 19:53:46 UTC
[50/50] [abbrv] beam git commit: This closes #3603
This closes #3603
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/ada24c05
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/ada24c05
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/ada24c05
Branch: refs/heads/DSL_SQL
Commit: ada24c059b1337fe02517c9f66fa9d29fb8bcc61
Parents: 152115e afeba37
Author: Jean-Baptiste Onofré <jb...@apache.org>
Authored: Thu Jul 20 21:52:35 2017 +0200
Committer: Jean-Baptiste Onofré <jb...@apache.org>
Committed: Thu Jul 20 21:52:35 2017 +0200
----------------------------------------------------------------------
.github/PULL_REQUEST_TEMPLATE.md | 16 +-
.gitignore | 2 +-
.../jenkins/common_job_properties.groovy | 9 +-
.../job_beam_PerformanceTests_Python.groovy | 58 +
..._beam_PostCommit_Java_JDKVersionsTest.groovy | 2 +
..._PostCommit_Java_MavenInstall_Windows.groovy | 3 +-
.../job_beam_PreCommit_Website_Merge.groovy | 59 +
README.md | 4 +-
examples/java/pom.xml | 32 +-
.../org/apache/beam/examples/WordCount.java | 4 +
.../examples/common/WriteOneFilePerWindow.java | 59 +-
.../apache/beam/examples/complete/TfIdf.java | 3 +-
.../examples/complete/TopWikipediaSessions.java | 24 +-
.../beam/examples/complete/TrafficRoutes.java | 19 +
.../beam/examples/cookbook/TriggerExample.java | 6 +-
.../beam/examples/DebuggingWordCountTest.java | 11 +-
.../beam/examples/WindowedWordCountIT.java | 4 +-
examples/java8/pom.xml | 20 +-
.../complete/game/utils/WriteToText.java | 49 +-
.../examples/complete/game/LeaderBoardTest.java | 2 +
examples/pom.xml | 2 +-
pom.xml | 127 +-
runners/apex/pom.xml | 20 +-
.../apache/beam/runners/apex/ApexRunner.java | 61 +-
.../translation/ApexPipelineTranslator.java | 16 +-
.../apex/translation/TranslationContext.java | 4 +-
.../operators/ApexParDoOperator.java | 21 +-
.../runners/apex/examples/WordCountTest.java | 8 +-
.../utils/ApexStateInternalsTest.java | 411 ++-----
runners/core-construction-java/pom.xml | 2 +-
.../CreatePCollectionViewTranslation.java | 15 +-
.../construction/ElementAndRestriction.java | 42 -
.../ElementAndRestrictionCoder.java | 88 --
.../construction/PCollectionTranslation.java | 16 +
.../core/construction/PTransformMatchers.java | 109 +-
.../construction/PTransformTranslation.java | 11 +-
.../core/construction/ParDoTranslation.java | 82 +-
.../construction/RunnerPCollectionView.java | 31 +-
.../core/construction/SplittableParDo.java | 124 +-
.../construction/TestStreamTranslation.java | 49 +-
.../core/construction/TransformInputs.java | 50 +
.../WindowingStrategyTranslation.java | 27 +-
.../construction/WriteFilesTranslation.java | 67 +-
.../construction/metrics/MetricFiltering.java | 102 ++
.../core/construction/metrics/MetricKey.java | 43 +
.../core/construction/metrics/package-info.java | 22 +
.../runners/core/metrics/MetricFiltering.java | 102 --
.../beam/runners/core/metrics/MetricKey.java | 43 -
.../beam/runners/core/metrics/package-info.java | 22 -
.../ElementAndRestrictionCoderTest.java | 126 --
.../PCollectionTranslationTest.java | 22 +
.../construction/PTransformMatchersTest.java | 54 +-
.../core/construction/ParDoTranslationTest.java | 28 +-
.../core/construction/SplittableParDoTest.java | 18 +-
.../core/construction/TransformInputsTest.java | 166 +++
.../WindowingStrategyTranslationTest.java | 3 +
.../construction/WriteFilesTranslationTest.java | 68 +-
.../metrics/MetricFilteringTest.java | 148 +++
.../core/metrics/MetricFilteringTest.java | 148 ---
runners/core-java/pom.xml | 2 +-
.../runners/core/InMemoryTimerInternals.java | 9 +
.../core/LateDataDroppingDoFnRunner.java | 33 +-
...eBoundedSplittableProcessElementInvoker.java | 40 +-
.../beam/runners/core/ProcessFnRunner.java | 16 +-
.../beam/runners/core/ReduceFnRunner.java | 135 ++-
.../beam/runners/core/SimpleDoFnRunner.java | 20 +
.../core/SplittableParDoViaKeyedWorkItems.java | 58 +-
.../core/SplittableProcessElementInvoker.java | 25 +-
.../org/apache/beam/runners/core/StateTags.java | 3 +
.../beam/runners/core/SystemReduceFn.java | 6 +
.../runners/core/metrics/MetricUpdates.java | 1 +
.../core/metrics/MetricsContainerImpl.java | 1 +
.../core/metrics/MetricsContainerStepMap.java | 2 +
.../core/triggers/AfterAllStateMachine.java | 25 +-
.../AfterDelayFromFirstElementStateMachine.java | 6 +-
.../core/triggers/AfterFirstStateMachine.java | 20 +-
.../core/triggers/AfterPaneStateMachine.java | 6 +-
.../triggers/AfterWatermarkStateMachine.java | 7 +-
.../triggers/ExecutableTriggerStateMachine.java | 23 +-
.../core/triggers/NeverStateMachine.java | 5 +-
.../core/triggers/TriggerStateMachine.java | 27 -
.../core/InMemoryStateInternalsTest.java | 569 +--------
...ndedSplittableProcessElementInvokerTest.java | 47 +-
.../beam/runners/core/ReduceFnRunnerTest.java | 374 +++++-
.../beam/runners/core/ReduceFnTester.java | 48 +-
.../core/SplittableParDoProcessFnTest.java | 117 +-
.../beam/runners/core/StateInternalsTest.java | 613 ++++++++++
.../beam/runners/core/WindowMatchers.java | 15 +
.../triggers/AfterFirstStateMachineTest.java | 5 +-
.../AfterWatermarkStateMachineTest.java | 7 +-
.../core/triggers/StubTriggerStateMachine.java | 7 +-
runners/direct-java/pom.xml | 7 +-
.../beam/runners/direct/CommittedResult.java | 12 +-
.../apache/beam/runners/direct/DirectGraph.java | 38 +-
.../beam/runners/direct/DirectGraphVisitor.java | 48 +-
.../beam/runners/direct/DirectGroupByKey.java | 13 +-
.../direct/DirectGroupByKeyOverrideFactory.java | 14 +-
.../beam/runners/direct/DirectMetrics.java | 4 +-
.../beam/runners/direct/DirectRegistrar.java | 2 +-
.../beam/runners/direct/DirectRunner.java | 64 +-
.../beam/runners/direct/DirectTestOptions.java | 42 +
.../runners/direct/DirectTimerInternals.java | 9 +
.../beam/runners/direct/EvaluationContext.java | 26 +-
.../direct/ExecutorServiceParallelExecutor.java | 27 +-
.../runners/direct/ParDoEvaluatorFactory.java | 9 +-
.../direct/ParDoMultiOverrideFactory.java | 121 +-
...littableProcessElementsEvaluatorFactory.java | 37 +-
.../direct/StatefulParDoEvaluatorFactory.java | 12 +-
.../direct/TestStreamEvaluatorFactory.java | 20 +-
.../runners/direct/ViewEvaluatorFactory.java | 8 +-
.../runners/direct/ViewOverrideFactory.java | 69 +-
.../beam/runners/direct/WatermarkManager.java | 18 +-
.../direct/WriteWithShardingFactory.java | 34 +-
.../runners/direct/CommittedResultTest.java | 17 +-
.../runners/direct/DirectGraphVisitorTest.java | 10 +-
.../beam/runners/direct/DirectGraphs.java | 7 +
.../beam/runners/direct/DirectMetricsTest.java | 2 +-
.../runners/direct/DirectRegistrarTest.java | 2 +-
.../runners/direct/EvaluationContextTest.java | 7 +-
.../ImmutabilityEnforcementFactoryTest.java | 4 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 7 +-
.../StatefulParDoEvaluatorFactoryTest.java | 65 +-
.../runners/direct/TransformExecutorTest.java | 12 +-
.../direct/ViewEvaluatorFactoryTest.java | 8 +-
.../runners/direct/ViewOverrideFactoryTest.java | 37 +-
.../direct/WatermarkCallbackExecutorTest.java | 1 +
.../runners/direct/WatermarkManagerTest.java | 16 +-
.../direct/WriteWithShardingFactoryTest.java | 44 +-
runners/flink/pom.xml | 11 +-
.../runners/flink/CreateStreamingFlinkView.java | 154 +++
.../flink/FlinkBatchTranslationContext.java | 3 +-
.../FlinkPipelineExecutionEnvironment.java | 2 +
.../flink/FlinkStreamingPipelineTranslator.java | 86 +-
.../FlinkStreamingTransformTranslators.java | 36 +-
.../flink/FlinkStreamingTranslationContext.java | 3 +-
.../flink/FlinkStreamingViewOverrides.java | 372 ------
.../runners/flink/FlinkTransformOverrides.java | 53 +
.../wrappers/streaming/DoFnOperator.java | 9 +
.../streaming/SplittableDoFnOperator.java | 16 +-
.../streaming/state/FlinkStateInternals.java | 425 +++----
.../FlinkBroadcastStateInternalsTest.java | 242 +---
.../FlinkKeyGroupStateInternalsTest.java | 359 +++---
.../streaming/FlinkSplitStateInternalsTest.java | 132 ++-
.../streaming/FlinkStateInternalsTest.java | 343 +-----
runners/google-cloud-dataflow-java/pom.xml | 10 +-
.../dataflow/BatchStatefulParDoOverrides.java | 4 +
.../runners/dataflow/BatchViewOverrides.java | 182 +--
.../runners/dataflow/CreateDataflowView.java | 8 +-
.../beam/runners/dataflow/DataflowMetrics.java | 306 +++--
.../runners/dataflow/DataflowPipelineJob.java | 18 +-
.../dataflow/DataflowPipelineTranslator.java | 62 +-
.../beam/runners/dataflow/DataflowRunner.java | 139 ++-
.../dataflow/SplittableParDoOverrides.java | 76 ++
.../dataflow/StreamingViewOverrides.java | 10 +-
.../runners/dataflow/TransformTranslator.java | 4 +-
.../DataflowPipelineWorkerPoolOptions.java | 3 +
.../runners/dataflow/util/MonitoringUtil.java | 16 +-
.../runners/dataflow/util/PropertyNames.java | 1 +
.../beam/runners/dataflow/util/TimeUtil.java | 24 +-
.../BatchStatefulParDoOverridesTest.java | 1 +
.../runners/dataflow/DataflowMetricsTest.java | 215 +++-
.../DataflowPipelineTranslatorTest.java | 96 +-
.../runners/dataflow/DataflowRunnerTest.java | 198 +++-
.../runners/dataflow/util/TimeUtilTest.java | 6 +
runners/pom.xml | 2 +-
runners/spark/pom.xml | 70 +-
.../spark/SparkNativePipelineVisitor.java | 3 +-
.../apache/beam/runners/spark/SparkRunner.java | 9 +-
.../beam/runners/spark/TestSparkRunner.java | 2 +-
.../SparkGroupAlsoByWindowViaWindowSet.java | 6 +-
.../spark/stateful/SparkTimerInternals.java | 18 +-
.../spark/translation/EvaluationContext.java | 4 +-
.../spark/translation/TransformTranslator.java | 50 +-
.../spark/util/GlobalWatermarkHolder.java | 127 +-
.../spark/GlobalWatermarkHolderTest.java | 18 +-
.../runners/spark/SparkRunnerDebuggerTest.java | 26 +-
.../spark/stateful/SparkStateInternalsTest.java | 66 ++
.../spark/translation/StorageLevelTest.java | 4 +-
sdks/common/fn-api/pom.xml | 2 +-
.../fn-api/src/main/proto/beam_fn_api.proto | 237 +---
sdks/common/pom.xml | 2 +-
sdks/common/runner-api/pom.xml | 2 +-
.../src/main/proto/beam_runner_api.proto | 26 +-
sdks/java/build-tools/pom.xml | 2 +-
.../src/main/resources/beam/checkstyle.xml | 8 +
.../src/main/resources/beam/findbugs-filter.xml | 9 +
sdks/java/core/pom.xml | 2 +-
.../java/org/apache/beam/sdk/coders/Coder.java | 12 +-
.../apache/beam/sdk/coders/CoderRegistry.java | 9 +
.../apache/beam/sdk/coders/ShardedKeyCoder.java | 66 ++
.../java/org/apache/beam/sdk/io/AvroIO.java | 220 ++--
.../java/org/apache/beam/sdk/io/AvroSink.java | 32 +-
.../java/org/apache/beam/sdk/io/AvroSource.java | 6 -
.../apache/beam/sdk/io/CompressedSource.java | 40 +-
.../beam/sdk/io/DefaultFilenamePolicy.java | 326 ++++--
.../beam/sdk/io/DynamicFileDestinations.java | 115 ++
.../org/apache/beam/sdk/io/FileBasedSink.java | 684 +++++------
.../apache/beam/sdk/io/OffsetBasedSource.java | 22 +-
.../java/org/apache/beam/sdk/io/TFRecordIO.java | 44 +-
.../java/org/apache/beam/sdk/io/TextIO.java | 713 +++++++++---
.../java/org/apache/beam/sdk/io/TextSink.java | 22 +-
.../java/org/apache/beam/sdk/io/WriteFiles.java | 704 +++++++----
.../beam/sdk/io/range/ByteKeyRangeTracker.java | 22 +-
.../apache/beam/sdk/io/range/OffsetRange.java | 101 ++
.../beam/sdk/io/range/OffsetRangeTracker.java | 3 +
.../sdk/options/PipelineOptionsFactory.java | 18 +-
.../sdk/options/PipelineOptionsValidator.java | 34 +-
.../sdk/options/ProxyInvocationHandler.java | 19 +-
.../beam/sdk/runners/TransformHierarchy.java | 165 ++-
.../org/apache/beam/sdk/testing/PAssert.java | 5 +-
.../apache/beam/sdk/testing/StaticWindows.java | 5 +
.../apache/beam/sdk/testing/StreamingIT.java | 4 +
.../apache/beam/sdk/testing/TestPipeline.java | 63 +-
.../org/apache/beam/sdk/testing/TestStream.java | 12 +
.../org/apache/beam/sdk/transforms/Combine.java | 31 +-
.../org/apache/beam/sdk/transforms/DoFn.java | 55 +-
.../apache/beam/sdk/transforms/DoFnTester.java | 21 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 41 +-
.../sdk/transforms/SerializableFunctions.java | 50 +
.../org/apache/beam/sdk/transforms/View.java | 40 +-
.../reflect/ByteBuddyDoFnInvokerFactory.java | 27 +
.../reflect/ByteBuddyOnTimerInvokerFactory.java | 73 +-
.../sdk/transforms/reflect/DoFnInvoker.java | 17 +-
.../sdk/transforms/reflect/DoFnInvokers.java | 9 -
.../sdk/transforms/reflect/DoFnSignature.java | 33 +-
.../sdk/transforms/reflect/DoFnSignatures.java | 44 +-
.../reflect/OnTimerMethodSpecifier.java | 37 +
.../transforms/splittabledofn/OffsetRange.java | 77 --
.../splittabledofn/OffsetRangeTracker.java | 11 +
.../splittabledofn/RestrictionTracker.java | 11 +-
.../sdk/transforms/windowing/GlobalWindows.java | 5 +
.../windowing/PartitioningWindowFn.java | 5 +
.../transforms/windowing/SlidingWindows.java | 5 +
.../beam/sdk/transforms/windowing/Window.java | 32 +
.../beam/sdk/transforms/windowing/WindowFn.java | 11 +
.../apache/beam/sdk/util/IdentityWindowFn.java | 6 +-
.../org/apache/beam/sdk/values/PCollection.java | 12 +
.../beam/sdk/values/PCollectionViews.java | 37 +
.../java/org/apache/beam/sdk/values/PValue.java | 4 +-
.../org/apache/beam/sdk/values/PValueBase.java | 12 -
.../org/apache/beam/sdk/values/ShardedKey.java | 65 ++
.../beam/sdk/values/WindowingStrategy.java | 46 +-
.../beam/sdk/coders/DefaultCoderTest.java | 3 +-
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 366 +++---
.../beam/sdk/io/DefaultFilenamePolicyTest.java | 135 ++-
.../sdk/io/DrunkWritableByteChannelFactory.java | 2 +-
.../apache/beam/sdk/io/FileBasedSinkTest.java | 175 ++-
.../java/org/apache/beam/sdk/io/SimpleSink.java | 56 +-
.../org/apache/beam/sdk/io/TextIOReadTest.java | 847 ++++++++++++++
.../java/org/apache/beam/sdk/io/TextIOTest.java | 1095 +-----------------
.../org/apache/beam/sdk/io/TextIOWriteTest.java | 604 ++++++++++
.../org/apache/beam/sdk/io/WriteFilesTest.java | 504 ++++++--
.../beam/sdk/metrics/MetricResultsMatchers.java | 2 +-
.../options/PipelineOptionsValidatorTest.java | 44 +
.../sdk/options/ProxyInvocationHandlerTest.java | 19 +
.../sdk/runners/TransformHierarchyTest.java | 197 ++++
.../sdk/testing/PCollectionViewTesting.java | 8 +
.../beam/sdk/testing/TestPipelineTest.java | 38 +-
.../apache/beam/sdk/transforms/CombineTest.java | 365 ++++--
.../beam/sdk/transforms/DoFnTesterTest.java | 32 +
.../beam/sdk/transforms/GroupByKeyTest.java | 195 +++-
.../apache/beam/sdk/transforms/ParDoTest.java | 202 ++++
.../beam/sdk/transforms/SplittableDoFnTest.java | 155 ++-
.../transforms/reflect/DoFnInvokersTest.java | 93 +-
.../DoFnSignaturesProcessElementTest.java | 2 +-
.../DoFnSignaturesSplittableDoFnTest.java | 83 +-
.../transforms/reflect/DoFnSignaturesTest.java | 14 +
.../splittabledofn/OffsetRangeTrackerTest.java | 1 +
.../windowing/SlidingWindowsTest.java | 30 +-
.../google-cloud-platform-core/pom.xml | 2 +-
.../java/org/apache/beam/sdk/util/GcsUtil.java | 2 +-
.../sdk/util/RetryHttpRequestInitializer.java | 147 ++-
.../extensions/gcp/GcpCoreApiSurfaceTest.java | 48 +-
.../util/RetryHttpRequestInitializerTest.java | 31 +-
sdks/java/extensions/jackson/pom.xml | 2 +-
sdks/java/extensions/join-library/pom.xml | 2 +-
sdks/java/extensions/pom.xml | 2 +-
sdks/java/extensions/protobuf/pom.xml | 2 +-
sdks/java/extensions/sorter/pom.xml | 8 +-
sdks/java/harness/pom.xml | 18 +-
.../beam/fn/harness/BeamFnDataReadRunner.java | 173 +++
.../beam/fn/harness/BeamFnDataWriteRunner.java | 159 +++
.../beam/fn/harness/BoundedSourceRunner.java | 167 +++
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 548 +++++++++
.../fn/harness/PTransformRunnerFactory.java | 81 ++
.../harness/channel/ManagedChannelFactory.java | 6 +
.../harness/control/ProcessBundleHandler.java | 295 ++---
.../fn/harness/control/RegisterHandler.java | 2 +-
.../beam/runners/core/BeamFnDataReadRunner.java | 115 --
.../runners/core/BeamFnDataWriteRunner.java | 98 --
.../beam/runners/core/BoundedSourceRunner.java | 105 --
.../apache/beam/runners/core/package-info.java | 22 -
.../fn/harness/BeamFnDataReadRunnerTest.java | 281 +++++
.../fn/harness/BeamFnDataWriteRunnerTest.java | 269 +++++
.../fn/harness/BoundedSourceRunnerTest.java | 187 +++
.../beam/fn/harness/FnApiDoFnRunnerTest.java | 210 ++++
.../control/ProcessBundleHandlerTest.java | 521 ++-------
.../fn/harness/control/RegisterHandlerTest.java | 8 +-
.../runners/core/BeamFnDataReadRunnerTest.java | 195 ----
.../runners/core/BeamFnDataWriteRunnerTest.java | 163 ---
.../runners/core/BoundedSourceRunnerTest.java | 113 --
sdks/java/io/amqp/pom.xml | 100 ++
.../org/apache/beam/sdk/io/amqp/AmqpIO.java | 399 +++++++
.../beam/sdk/io/amqp/AmqpMessageCoder.java | 79 ++
.../amqp/AmqpMessageCoderProviderRegistrar.java | 44 +
.../apache/beam/sdk/io/amqp/package-info.java | 22 +
.../org/apache/beam/sdk/io/amqp/AmqpIOTest.java | 148 +++
.../beam/sdk/io/amqp/AmqpMessageCoderTest.java | 89 ++
sdks/java/io/cassandra/pom.xml | 2 +-
.../beam/sdk/io/cassandra/CassandraIO.java | 2 +-
sdks/java/io/common/pom.xml | 12 +-
.../sdk/io/common/IOTestPipelineOptions.java | 6 +-
.../org/apache/beam/sdk/io/common/TestRow.java | 114 ++
sdks/java/io/elasticsearch/pom.xml | 10 +-
.../sdk/io/elasticsearch/ElasticsearchIO.java | 17 +-
.../elasticsearch/ElasticSearchIOTestUtils.java | 81 +-
.../sdk/io/elasticsearch/ElasticsearchIOIT.java | 14 +-
.../io/elasticsearch/ElasticsearchIOTest.java | 36 +-
.../elasticsearch/ElasticsearchTestDataSet.java | 37 +-
sdks/java/io/google-cloud-platform/pom.xml | 14 +-
.../beam/sdk/io/gcp/bigquery/BatchLoads.java | 2 +
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 181 ++-
.../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 64 +-
.../io/gcp/bigquery/DynamicDestinations.java | 29 +-
.../io/gcp/bigquery/GenerateShardedTable.java | 1 +
.../io/gcp/bigquery/PassThroughThenCleanup.java | 46 +-
.../beam/sdk/io/gcp/bigquery/ShardedKey.java | 67 --
.../sdk/io/gcp/bigquery/ShardedKeyCoder.java | 74 --
.../sdk/io/gcp/bigquery/StreamingWriteFn.java | 1 +
.../io/gcp/bigquery/StreamingWriteTables.java | 2 +
.../sdk/io/gcp/bigquery/TagWithUniqueIds.java | 1 +
.../io/gcp/bigquery/WriteBundlesToFiles.java | 2 +
.../bigquery/WriteGroupedRecordsToFiles.java | 1 +
.../sdk/io/gcp/bigquery/WritePartition.java | 1 +
.../beam/sdk/io/gcp/bigquery/WriteTables.java | 1 +
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 8 +-
.../io/gcp/bigtable/BigtableServiceImpl.java | 9 +-
.../sdk/io/gcp/datastore/AdaptiveThrottler.java | 103 ++
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 194 +++-
.../sdk/io/gcp/datastore/MovingAverage.java | 50 +
.../sdk/io/gcp/spanner/AbstractSpannerFn.java | 58 +
.../sdk/io/gcp/spanner/CreateTransactionFn.java | 51 +
.../beam/sdk/io/gcp/spanner/MutationGroup.java | 67 ++
.../io/gcp/spanner/MutationSizeEstimator.java | 9 +
.../sdk/io/gcp/spanner/NaiveSpannerReadFn.java | 74 ++
.../beam/sdk/io/gcp/spanner/ReadOperation.java | 96 ++
.../beam/sdk/io/gcp/spanner/SpannerConfig.java | 137 +++
.../beam/sdk/io/gcp/spanner/SpannerIO.java | 715 +++++++++---
.../sdk/io/gcp/spanner/SpannerWriteGroupFn.java | 125 ++
.../beam/sdk/io/gcp/spanner/Transaction.java | 33 +
.../beam/sdk/io/gcp/GcpApiSurfaceTest.java | 10 +
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 76 +-
.../sdk/io/gcp/bigquery/FakeDatasetService.java | 5 +-
.../sdk/io/gcp/bigtable/BigtableReadIT.java | 5 +-
.../io/gcp/bigtable/BigtableTestOptions.java | 5 -
.../sdk/io/gcp/bigtable/BigtableWriteIT.java | 4 +-
.../io/gcp/datastore/AdaptiveThrottlerTest.java | 111 ++
.../sdk/io/gcp/datastore/DatastoreV1Test.java | 141 ++-
.../beam/sdk/io/gcp/datastore/V1TestUtil.java | 2 +-
.../sdk/io/gcp/spanner/FakeServiceFactory.java | 82 ++
.../gcp/spanner/MutationSizeEstimatorTest.java | 12 +
.../beam/sdk/io/gcp/spanner/RandomUtils.java | 41 +
.../sdk/io/gcp/spanner/SpannerIOReadTest.java | 332 ++++++
.../beam/sdk/io/gcp/spanner/SpannerIOTest.java | 244 ----
.../sdk/io/gcp/spanner/SpannerIOWriteTest.java | 258 +++++
.../beam/sdk/io/gcp/spanner/SpannerReadIT.java | 166 +++
.../beam/sdk/io/gcp/spanner/SpannerWriteIT.java | 26 +-
sdks/java/io/hadoop-common/pom.xml | 2 +-
sdks/java/io/hadoop-file-system/pom.xml | 33 +-
sdks/java/io/hadoop/input-format/pom.xml | 2 +-
.../hadoop/inputformat/HadoopInputFormatIO.java | 2 +-
sdks/java/io/hadoop/jdk1.8-tests/pom.xml | 4 +-
.../inputformat/HIFIOWithElasticTest.java | 11 +-
sdks/java/io/hadoop/pom.xml | 2 +-
sdks/java/io/hbase/pom.xml | 26 +-
.../io/hbase/HBaseCoderProviderRegistrar.java | 40 +
.../org/apache/beam/sdk/io/hbase/HBaseIO.java | 48 +-
.../beam/sdk/io/hbase/HBaseMutationCoder.java | 42 +
.../hbase/HBaseCoderProviderRegistrarTest.java | 45 +
.../apache/beam/sdk/io/hbase/HBaseIOTest.java | 49 +-
sdks/java/io/hcatalog/pom.xml | 175 +++
.../apache/beam/sdk/io/hcatalog/HCatalogIO.java | 492 ++++++++
.../beam/sdk/io/hcatalog/package-info.java | 22 +
.../io/hcatalog/EmbeddedMetastoreService.java | 87 ++
.../beam/sdk/io/hcatalog/HCatalogIOTest.java | 277 +++++
.../sdk/io/hcatalog/HCatalogIOTestUtils.java | 108 ++
.../hcatalog/src/test/resources/hive-site.xml | 301 +++++
sdks/java/io/jdbc/pom.xml | 14 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 2 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIOIT.java | 203 ++--
.../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 115 +-
.../beam/sdk/io/jdbc/JdbcTestDataSet.java | 130 ---
.../apache/beam/sdk/io/jdbc/JdbcTestHelper.java | 81 ++
sdks/java/io/jms/pom.xml | 2 +-
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 2 +-
sdks/java/io/kafka/pom.xml | 2 +-
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 132 ++-
.../apache/beam/sdk/io/kafka/KafkaIOTest.java | 30 +
sdks/java/io/kinesis/pom.xml | 2 +-
.../sdk/io/kinesis/CheckpointGenerator.java | 6 +-
.../beam/sdk/io/kinesis/CustomOptional.java | 111 +-
.../io/kinesis/DynamicCheckpointGenerator.java | 52 +-
.../sdk/io/kinesis/GetKinesisRecordsResult.java | 49 +-
.../sdk/io/kinesis/KinesisClientProvider.java | 4 +-
.../apache/beam/sdk/io/kinesis/KinesisIO.java | 281 ++---
.../beam/sdk/io/kinesis/KinesisReader.java | 206 ++--
.../sdk/io/kinesis/KinesisReaderCheckpoint.java | 97 +-
.../beam/sdk/io/kinesis/KinesisRecord.java | 177 +--
.../beam/sdk/io/kinesis/KinesisRecordCoder.java | 68 +-
.../beam/sdk/io/kinesis/KinesisSource.java | 147 +--
.../beam/sdk/io/kinesis/RecordFilter.java | 18 +-
.../apache/beam/sdk/io/kinesis/RoundRobin.java | 37 +-
.../beam/sdk/io/kinesis/ShardCheckpoint.java | 241 ++--
.../sdk/io/kinesis/ShardRecordsIterator.java | 106 +-
.../sdk/io/kinesis/SimplifiedKinesisClient.java | 215 ++--
.../beam/sdk/io/kinesis/StartingPoint.java | 84 +-
.../io/kinesis/StaticCheckpointGenerator.java | 27 +-
.../io/kinesis/TransientKinesisException.java | 7 +-
.../beam/sdk/io/kinesis/AmazonKinesisMock.java | 539 ++++-----
.../beam/sdk/io/kinesis/CustomOptionalTest.java | 27 +-
.../kinesis/DynamicCheckpointGeneratorTest.java | 33 +-
.../sdk/io/kinesis/KinesisMockReadTest.java | 97 +-
.../io/kinesis/KinesisReaderCheckpointTest.java | 52 +-
.../beam/sdk/io/kinesis/KinesisReaderIT.java | 127 +-
.../beam/sdk/io/kinesis/KinesisReaderTest.java | 166 +--
.../sdk/io/kinesis/KinesisRecordCoderTest.java | 34 +-
.../beam/sdk/io/kinesis/KinesisTestOptions.java | 43 +-
.../beam/sdk/io/kinesis/KinesisUploader.java | 70 +-
.../beam/sdk/io/kinesis/RecordFilterTest.java | 52 +-
.../beam/sdk/io/kinesis/RoundRobinTest.java | 42 +-
.../sdk/io/kinesis/ShardCheckpointTest.java | 203 ++--
.../io/kinesis/ShardRecordsIteratorTest.java | 216 ++--
.../io/kinesis/SimplifiedKinesisClientTest.java | 351 +++---
sdks/java/io/mongodb/pom.xml | 2 +-
.../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 2 +-
.../apache/beam/sdk/io/mongodb/MongoDbIO.java | 317 +++--
.../beam/sdk/io/mongodb/MongoDbIOTest.java | 37 +
sdks/java/io/mqtt/pom.xml | 2 +-
.../org/apache/beam/sdk/io/mqtt/MqttIO.java | 2 +-
sdks/java/io/pom.xml | 35 +-
sdks/java/io/xml/pom.xml | 2 +-
.../java/org/apache/beam/sdk/io/xml/XmlIO.java | 4 +-
.../org/apache/beam/sdk/io/xml/XmlSink.java | 21 +-
.../org/apache/beam/sdk/io/xml/XmlSinkTest.java | 4 +-
sdks/java/java8tests/pom.xml | 2 +-
sdks/java/javadoc/pom.xml | 19 +-
.../maven-archetypes/examples-java8/pom.xml | 2 +-
.../main/resources/archetype-resources/pom.xml | 1 -
sdks/java/maven-archetypes/examples/pom.xml | 2 +-
.../main/resources/archetype-resources/pom.xml | 1 -
sdks/java/maven-archetypes/pom.xml | 2 +-
sdks/java/maven-archetypes/starter/pom.xml | 2 +-
.../resources/projects/basic/reference/pom.xml | 2 +-
sdks/java/pom.xml | 2 +-
sdks/pom.xml | 2 +-
sdks/python/apache_beam/coders/coder_impl.py | 4 +
sdks/python/apache_beam/coders/coders.py | 7 +-
.../apache_beam/coders/coders_test_common.py | 8 +
.../apache_beam/examples/snippets/snippets.py | 2 +-
.../examples/snippets/snippets_test.py | 16 +
.../apache_beam/examples/streaming_wordcount.py | 25 +-
.../apache_beam/examples/windowed_wordcount.py | 93 ++
sdks/python/apache_beam/io/filesystem.py | 22 +-
sdks/python/apache_beam/io/gcp/bigquery.py | 100 +-
sdks/python/apache_beam/io/gcp/bigquery_test.py | 105 +-
.../io/gcp/datastore/v1/datastoreio.py | 84 +-
.../io/gcp/datastore/v1/datastoreio_test.py | 53 +-
.../apache_beam/io/gcp/datastore/v1/helper.py | 43 +-
.../io/gcp/datastore/v1/helper_test.py | 22 +-
.../apache_beam/io/gcp/datastore/v1/util.py | 95 ++
.../io/gcp/datastore/v1/util_test.py | 67 ++
sdks/python/apache_beam/io/gcp/gcsio.py | 10 +-
sdks/python/apache_beam/io/gcp/pubsub.py | 180 ++-
sdks/python/apache_beam/io/gcp/pubsub_test.py | 101 +-
.../io/gcp/tests/bigquery_matcher.py | 6 +-
.../io/gcp/tests/bigquery_matcher_test.py | 2 +-
sdks/python/apache_beam/io/range_trackers.py | 130 ---
.../apache_beam/io/range_trackers_test.py | 186 ---
.../apache_beam/options/pipeline_options.py | 35 +-
.../options/pipeline_options_test.py | 39 +-
.../apache_beam/options/value_provider_test.py | 93 +-
sdks/python/apache_beam/pipeline.py | 230 +++-
sdks/python/apache_beam/pipeline_test.py | 53 +
sdks/python/apache_beam/portability/__init__.py | 18 +
.../apache_beam/portability/api/__init__.py | 21 +
sdks/python/apache_beam/pvalue.py | 2 +-
sdks/python/apache_beam/runners/api/__init__.py | 21 -
.../runners/dataflow/dataflow_runner.py | 112 +-
.../runners/dataflow/dataflow_runner_test.py | 24 +-
.../runners/dataflow/internal/apiclient.py | 42 +-
.../runners/dataflow/internal/apiclient_test.py | 29 +-
.../runners/dataflow/internal/dependency.py | 69 +-
.../runners/dataflow/native_io/iobase_test.py | 39 +-
.../dataflow/native_io/streaming_create.py | 72 ++
.../runners/dataflow/ptransform_overrides.py | 52 +
.../runners/dataflow/test_dataflow_runner.py | 5 +-
.../runners/direct/bundle_factory.py | 2 +-
.../apache_beam/runners/direct/direct_runner.py | 108 ++
.../runners/direct/evaluation_context.py | 73 +-
.../apache_beam/runners/direct/executor.py | 135 ++-
.../runners/direct/transform_evaluator.py | 447 ++++++-
.../runners/direct/transform_result.py | 41 -
sdks/python/apache_beam/runners/direct/util.py | 67 ++
.../runners/direct/watermark_manager.py | 100 +-
.../apache_beam/runners/pipeline_context.py | 19 +-
.../runners/portability/fn_api_runner.py | 306 +++--
.../runners/portability/fn_api_runner_test.py | 31 +-
.../runners/worker/bundle_processor.py | 426 +++++++
.../apache_beam/runners/worker/data_plane.py | 36 +-
.../runners/worker/data_plane_test.py | 2 +-
.../apache_beam/runners/worker/log_handler.py | 2 +-
.../runners/worker/log_handler_test.py | 2 +-
.../runners/worker/operation_specs.py | 9 +-
.../apache_beam/runners/worker/operations.py | 1 +
.../apache_beam/runners/worker/sdk_worker.py | 370 +-----
.../runners/worker/sdk_worker_main.py | 2 +-
.../runners/worker/sdk_worker_test.py | 95 +-
sdks/python/apache_beam/testing/test_stream.py | 5 +
.../apache_beam/testing/test_stream_test.py | 68 ++
sdks/python/apache_beam/transforms/combiners.py | 8 +
.../apache_beam/transforms/combiners_test.py | 7 +-
sdks/python/apache_beam/transforms/core.py | 104 +-
.../python/apache_beam/transforms/ptransform.py | 43 +-
sdks/python/apache_beam/transforms/trigger.py | 49 +-
sdks/python/apache_beam/transforms/window.py | 4 +-
.../apache_beam/typehints/trivial_inference.py | 3 +-
.../typehints/trivial_inference_test.py | 7 +
sdks/python/apache_beam/utils/plugin.py | 42 +
sdks/python/apache_beam/utils/timestamp.py | 5 +
sdks/python/apache_beam/utils/urns.py | 2 +-
sdks/python/apache_beam/version.py | 2 +-
sdks/python/gen_protos.py | 2 +-
sdks/python/pom.xml | 2 +-
sdks/python/run_pylint.sh | 2 +-
sdks/python/setup.py | 5 +-
535 files changed, 26736 insertions(+), 13977 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/ada24c05/pom.xml
----------------------------------------------------------------------