You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2017/08/07 23:54:02 UTC
[37/50] [abbrv] beam git commit: This closes #3479: [BEAM-79] Merge
master into gearpump-runner branch
This closes #3479: [BEAM-79] Merge master into gearpump-runner branch
Don't call .testingPipelineOptions() a second time
GCP IO ITs now all use --project option
Select SDK distribution based on the selected SDK name
[BEAM-2373] Upgrade commons-compress dependency version to 1.14
Define the projectId in the SpannerIO Read Test (utest, not itest)
Use SDK harness container for FnAPI jobs when worker_harness_container_image is not specified. Add a separate image tag to use with the SDK harness container.
Ditch apache commons
Add PubSub I/O support to Python DirectRunner
Only use ASCII 'a' through 'z' for temporary Spanner tables
ReduceFnRunner.onTrigger: add short circuit for empty pane, and move inputWM and pane after the short circuit.
WindowingStrategy: add OnTimeBehavior to control whether to emit empty ON_TIME pane.
Removed OnceTriggerStateMachine
Visit composite nodes when checking for picklability.
Upgrade beam bigtable client dependency to 0.9.7.1
Add a Combine Test for Sliding Windows without Context
[BEAM-2389] moved GcpCoreApiSurfaceTest to corresponding module, adapted exposed packagees
Add Experimental annotation to AMQP and refine Kind for the Experimental IOs
[BEAM-2488] Elasticsearch IO should read also in replica shards
Use PCollectionViews.toAdditionalInputs in Combine
Use PCollectionViews.toAdditionalInputs in ParDo
Use PCollectionViews.toAdditionalInputs in ParDoMultiOverrideFactory
Fix getAdditionalInputs for SplittableParDo transforms
Add utility to expand list of PCollectionViews
Read api with naive implementation
Pre read api refactoring. Extract `SpannerConfig` and `AbstractSpannerFn`
Bump spanner version
[BEAM-1187] Improve logging to contain the number of retries done due to IOException and unsuccessful response codes.
Add WindowFn#assignsToOneWindow
Use installed distribution name for sdk name
[BEAM-2522] upgrading jackson to 2.8.9 (mitigating #1599)
Enable grpc controller in fn_api_runner
Removed uses of proto builder clone method
[BEAM-2514] Improve error message on missing required value
[BEAM-1237] Create AmqpIO
Implement streaming GroupByKey in Python DirectRunner
Bump Dataflow worker to 0623
Reintroduces DoFn.ProcessContinuation (Dataflow worker compatibility part)
Remove old deprecated PubSub code
Fix a typo in function args
Avoid pickling the entire pipeline per-transform.
Fix python fn API data plane remote grpc port access
[BEAM-2745] Add Jenkins Suite for Python Performance Test
[BEAM-2489] Use dynamic ES port in HIFIOWithElasticTest
[BEAM-2497] Fix the reading of concat gzip files
Allow output from FinishBundle in DoFnTester
DataflowRunner: Reject merging windowing for stateful ParDo
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/627ae0bc
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/627ae0bc
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/627ae0bc
Branch: refs/heads/master
Commit: 627ae0bc3912ace4b5acf5b8e9fd779ad4461cf7
Parents: 99f4f8b f158257
Author: Kenneth Knowles <kl...@google.com>
Authored: Fri Jun 30 14:30:25 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Fri Jun 30 14:30:25 2017 -0700
----------------------------------------------------------------------
.../jenkins/common_job_properties.groovy | 4 +-
.../job_beam_PerformanceTests_Python.groovy | 58 ++
pom.xml | 20 +-
.../apache/beam/runners/apex/ApexRunner.java | 2 +-
.../core/construction/SplittableParDo.java | 66 +-
.../WindowingStrategyTranslation.java | 26 +-
.../core/construction/SplittableParDoTest.java | 8 +-
.../beam/runners/core/ReduceFnRunner.java | 20 +-
.../core/triggers/AfterAllStateMachine.java | 25 +-
.../AfterDelayFromFirstElementStateMachine.java | 6 +-
.../core/triggers/AfterFirstStateMachine.java | 20 +-
.../core/triggers/AfterPaneStateMachine.java | 6 +-
.../triggers/AfterWatermarkStateMachine.java | 7 +-
.../triggers/ExecutableTriggerStateMachine.java | 23 +-
.../core/triggers/NeverStateMachine.java | 5 +-
.../core/triggers/TriggerStateMachine.java | 27 -
.../beam/runners/core/ReduceFnRunnerTest.java | 161 +++++
.../triggers/AfterFirstStateMachineTest.java | 5 +-
.../AfterWatermarkStateMachineTest.java | 7 +-
.../core/triggers/StubTriggerStateMachine.java | 7 +-
.../direct/ParDoMultiOverrideFactory.java | 16 +-
.../flink/FlinkStreamingPipelineTranslator.java | 2 +-
runners/google-cloud-dataflow-java/pom.xml | 2 +-
.../dataflow/BatchStatefulParDoOverrides.java | 2 +
.../dataflow/DataflowPipelineTranslator.java | 5 +-
.../beam/runners/dataflow/DataflowRunner.java | 10 +
.../dataflow/SplittableParDoOverrides.java | 2 +-
.../runners/dataflow/DataflowRunnerTest.java | 38 ++
.../src/main/proto/beam_runner_api.proto | 14 +
.../sdk/options/PipelineOptionsFactory.java | 18 +-
.../sdk/options/PipelineOptionsValidator.java | 34 +-
.../sdk/options/ProxyInvocationHandler.java | 4 +
.../apache/beam/sdk/testing/StaticWindows.java | 5 +
.../org/apache/beam/sdk/transforms/Combine.java | 13 +-
.../org/apache/beam/sdk/transforms/DoFn.java | 3 +
.../apache/beam/sdk/transforms/DoFnTester.java | 16 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 14 +-
.../reflect/ByteBuddyDoFnInvokerFactory.java | 6 +
.../sdk/transforms/reflect/DoFnInvoker.java | 4 +-
.../sdk/transforms/windowing/GlobalWindows.java | 5 +
.../windowing/PartitioningWindowFn.java | 5 +
.../transforms/windowing/SlidingWindows.java | 5 +
.../beam/sdk/transforms/windowing/Window.java | 32 +
.../beam/sdk/transforms/windowing/WindowFn.java | 11 +
.../apache/beam/sdk/util/IdentityWindowFn.java | 5 +
.../beam/sdk/values/PCollectionViews.java | 14 +
.../beam/sdk/values/WindowingStrategy.java | 46 +-
.../options/PipelineOptionsValidatorTest.java | 44 ++
.../sdk/options/ProxyInvocationHandlerTest.java | 7 +
.../apache/beam/sdk/transforms/CombineTest.java | 63 ++
.../beam/sdk/transforms/DoFnTesterTest.java | 32 +
.../windowing/SlidingWindowsTest.java | 30 +-
.../sdk/util/RetryHttpRequestInitializer.java | 148 +++--
.../extensions/gcp/GcpCoreApiSurfaceTest.java | 48 +-
.../util/RetryHttpRequestInitializerTest.java | 31 +-
sdks/java/io/amqp/pom.xml | 100 +++
.../org/apache/beam/sdk/io/amqp/AmqpIO.java | 399 ++++++++++++
.../beam/sdk/io/amqp/AmqpMessageCoder.java | 79 +++
.../amqp/AmqpMessageCoderProviderRegistrar.java | 44 ++
.../apache/beam/sdk/io/amqp/package-info.java | 22 +
.../org/apache/beam/sdk/io/amqp/AmqpIOTest.java | 148 +++++
.../beam/sdk/io/amqp/AmqpMessageCoderTest.java | 89 +++
.../beam/sdk/io/cassandra/CassandraIO.java | 2 +-
sdks/java/io/elasticsearch/pom.xml | 8 +
.../sdk/io/elasticsearch/ElasticsearchIO.java | 13 +-
sdks/java/io/google-cloud-platform/pom.xml | 13 +-
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 6 +-
.../io/gcp/bigtable/BigtableServiceImpl.java | 9 +-
.../sdk/io/gcp/spanner/AbstractSpannerFn.java | 58 ++
.../sdk/io/gcp/spanner/CreateTransactionFn.java | 51 ++
.../sdk/io/gcp/spanner/NaiveSpannerReadFn.java | 65 ++
.../beam/sdk/io/gcp/spanner/SpannerConfig.java | 137 +++++
.../beam/sdk/io/gcp/spanner/SpannerIO.java | 604 +++++++++++++------
.../sdk/io/gcp/spanner/SpannerWriteGroupFn.java | 125 ++++
.../beam/sdk/io/gcp/spanner/Transaction.java | 33 +
.../beam/sdk/io/gcp/GcpApiSurfaceTest.java | 10 +
.../sdk/io/gcp/bigtable/BigtableReadIT.java | 5 +-
.../io/gcp/bigtable/BigtableTestOptions.java | 5 -
.../sdk/io/gcp/bigtable/BigtableWriteIT.java | 4 +-
.../sdk/io/gcp/datastore/DatastoreV1Test.java | 4 +-
.../beam/sdk/io/gcp/datastore/V1TestUtil.java | 2 +-
.../sdk/io/gcp/spanner/FakeServiceFactory.java | 82 +++
.../beam/sdk/io/gcp/spanner/RandomUtils.java | 41 ++
.../sdk/io/gcp/spanner/SpannerIOReadTest.java | 281 +++++++++
.../beam/sdk/io/gcp/spanner/SpannerIOTest.java | 317 ----------
.../sdk/io/gcp/spanner/SpannerIOWriteTest.java | 258 ++++++++
.../beam/sdk/io/gcp/spanner/SpannerReadIT.java | 166 +++++
.../beam/sdk/io/gcp/spanner/SpannerWriteIT.java | 27 +-
.../hadoop/inputformat/HadoopInputFormatIO.java | 2 +-
.../inputformat/HIFIOWithElasticTest.java | 11 +-
.../org/apache/beam/sdk/io/hbase/HBaseIO.java | 2 +-
.../apache/beam/sdk/io/hcatalog/HCatalogIO.java | 2 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 2 +-
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 2 +-
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 2 +-
.../apache/beam/sdk/io/kinesis/KinesisIO.java | 2 +-
.../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 2 +-
.../apache/beam/sdk/io/mongodb/MongoDbIO.java | 2 +-
.../org/apache/beam/sdk/io/mqtt/MqttIO.java | 2 +-
sdks/java/io/pom.xml | 1 +
.../examples/snippets/snippets_test.py | 16 +
.../apache_beam/examples/streaming_wordcount.py | 14 +-
sdks/python/apache_beam/io/filesystem.py | 8 +
sdks/python/apache_beam/io/gcp/pubsub.py | 162 +++--
sdks/python/apache_beam/io/gcp/pubsub_test.py | 89 +--
sdks/python/apache_beam/pipeline.py | 12 +-
sdks/python/apache_beam/pipeline_test.py | 18 +
.../runners/dataflow/dataflow_runner.py | 11 +-
.../runners/dataflow/internal/apiclient.py | 6 +-
.../runners/dataflow/internal/dependency.py | 69 ++-
.../apache_beam/runners/direct/direct_runner.py | 83 ++-
.../runners/direct/evaluation_context.py | 2 +-
.../runners/direct/transform_evaluator.py | 227 ++++++-
sdks/python/apache_beam/runners/direct/util.py | 25 +-
.../runners/direct/watermark_manager.py | 26 +-
.../runners/portability/fn_api_runner.py | 12 +-
.../runners/portability/fn_api_runner_test.py | 23 +-
.../apache_beam/runners/worker/data_plane.py | 10 +-
.../apache_beam/runners/worker/sdk_worker.py | 2 +-
.../apache_beam/testing/test_stream_test.py | 37 +-
sdks/python/apache_beam/transforms/trigger.py | 16 +
121 files changed, 4273 insertions(+), 1071 deletions(-)
----------------------------------------------------------------------