You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2020/12/17 03:03:35 UTC
[beam] tag nightly-master updated (f137246 -> b8c001c)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/beam.git.
*** WARNING: tag nightly-master was modified! ***
from f137246 (commit)
to b8c001c (commit)
from f137246 Merge pull request #13556 [BEAM-11472] Support SDF on non-splitting runners such as legacy Dataflow.
add 31dcd53 [BEAM-10114] Convert PubsubLiteIO read to use SplittableDoFn.
add ec3efe2 Fix test.
add d3336fe fix: Testing issues
add 8892c1c [BEAM-10114] Convert PubsubLiteIO to use an SDF for reads.
add f5c6108 [BEAM-10114] Add byte and time limiting
add 6d8c0a9 [BEAM-10114] Add byte and time limiting
add 762b7ba [BEAM-10114] Add byte and time limiting
add 65b2c42 [BEAM-10114] Add byte and time limiting
add 418b3ac [BEAM-10114] Add byte and time limiting
add 843fc4a [BEAM-10114] Add byte and time limiting
add fa3fc2d fix: Linter issues
add ea33f9e fix: linters
add 88a3021 fix: linters
add b731da8 fix: Remove unnecessary shuffle.
add a432e8f fix: Address comments
add f87c984 Merge pull request #13470 from [BEAM-10114] Convert PubsubLiteIO read to use SplittableDoFn.
add dee8d49 [BEAM-11443] Implement byte-based GBK for Flink Streaming Runner.
add 56d4ac9 Merge pull request #13538: [BEAM-11443] Implement byte-based GBK for Flink Streaming Runner.
add ded0325 [BEAM-11075] Go SDK SideInput load tests (#13436)
add f1c1514 [BEAM-11267] Remove unecessary reshuffle for stateful ParDo after keyed operation.
add 76c6129 Merge pull request #13353: [BEAM-11267] Remove unnecessary reshuffle for stateful ParDo after key…
add 12b6015 Update DoFn docstring (#13559)
add cf20794 Fix concat source fraction_consumed() in degenerate case.
add 8f79065 Merge pull request #13563 from robertwb/concat-isdone
add b2f4784 [BEAM-11361] Dynamic splitting of dataframe csv reads. (#13488)
add 97ad770 fix PerPartitionSubscriptionSdf methods with incorrect visibility
add 92209d1 Merge pull request #13566 from fix PerPartitionSubscriptionSdf methods with incorrect visibility
add 1ccdee8 [BEAM-11432] put latest tag when publishing SDK head containers to gcr (#13517)
add 8de3288 [BEAM-10925] Add interface for SQL Java scalar UDF. (#13305)
add 554d254 Track transform processing thread in Java SDK harness and set log entry transform id field.
add a6ba301 Add javadoc
add 23b2007 Add tracking to other ProcessElement methods
add 945e794 Use LoadingCache instead of ConcurrentHashMap to limit the size for thread tracker
add 857d41f Address comment
add c8d9d1d Fix checkstyle
add 6869dfa Merge pull request #13533 from [BEAM-11474] Track transform processing thread in Java SDK harness and set log entry field
add 3de4f88 [BEAM-10925] Add SQL AggregateFn interface.
add a7457c0 Merge pull request #13306 from ibzib/aggregate-udf
add 3cacd4a [BEAM-11407] Add IT test to Bigtable for BeamSQL
add 1a5505b Merge pull request #13512 from piotr-szuberski/bigtable-it
add 8fa3e87 Thread safe addition of new metrics
add c7efdc1 Thread safe addition of new metrics
add 5f253b6 Merge pull request #13526 from [BEAM-11459] Thread safe addition of new metrics
add b8c001c Moving to 2.28.0-SNAPSHOT on master branch.
No new revisions were added by this update.
Summary of changes:
.../job_LoadTests_SideInput_Flink_Go.groovy | 103 +++++++
.../jenkins/job_LoadTests_SideInput_Go.groovy | 94 ++++++
.../jenkins/job_Publish_SDK_Image_Snapshots.groovy | 8 +-
.../perftests_metrics/SideInput_Load_Tests.json | 2 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 15 +-
gradle.properties | 4 +-
.../FlinkStreamingPortablePipelineTranslator.java | 27 +-
.../flink/FlinkStreamingTransformTranslators.java | 215 +++++++++----
.../flink/FlinkStreamingTranslationContext.java | 24 +-
.../wrappers/streaming/DoFnOperator.java | 5 +-
.../streaming/SingletonKeyedWorkItemCoder.java | 13 +-
.../wrappers/streaming/WorkItemKeySelector.java | 13 +-
.../FlinkStreamingPipelineTranslatorTest.java | 109 +++++++
.../wrappers/streaming/WindowDoFnOperatorTest.java | 5 +-
sdks/go/container/build.gradle | 2 +
sdks/go/test/load/build.gradle | 1 +
sdks/go/test/load/sideinput/sideinput.go | 100 +++++++
sdks/java/container/common.gradle | 2 +
.../provider/bigtable/BigtableClientWrapper.java | 115 +++++++
.../BigtableTableCreationFailuresTest.java | 2 +-
.../provider/bigtable/BigtableTableFlatTest.java | 110 +++----
.../meta/provider/bigtable/BigtableTableIT.java | 200 +++++++++++++
.../meta/provider/bigtable/BigtableTableTest.java | 94 ------
.../provider/bigtable/BigtableTableTestUtils.java | 237 +++++++++++++++
.../bigtable/BigtableTableWithRowsTest.java | 120 ++++----
.../sql/udf/build.gradle} | 22 +-
.../beam/sdk/extensions/sql/udf/AggregateFn.java | 70 +++++
.../beam/sdk/extensions/sql/udf/ScalarFn.java | 53 ++++
.../beam/sdk/extensions/sql/udf/package-info.java} | 15 +-
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 27 ++
.../harness/TransformProcessingThreadTracker.java | 49 +++
.../fn/harness/logging/BeamFnLoggingClient.java | 8 +
...fsetFinalizer.java => InitialOffsetReader.java} | 12 +-
.../io/gcp/pubsublite/InitialOffsetReaderImpl.java | 54 ++++
.../gcp/pubsublite/LimitingTopicBacklogReader.java | 81 +++++
...ffsetFinalizer.java => OffsetByteProgress.java} | 17 +-
.../io/gcp/pubsublite/OffsetByteRangeTracker.java | 172 +++++++++++
.../io/gcp/pubsublite/OffsetCheckpointMark.java | 74 -----
.../pubsublite/PerSubscriptionPartitionSdf.java | 110 +++++++
.../beam/sdk/io/gcp/pubsublite/PublisherCache.java | 2 +-
.../beam/sdk/io/gcp/pubsublite/PubsubLiteIO.java | 14 +-
.../gcp/pubsublite/PubsubLiteUnboundedReader.java | 333 ---------------------
.../gcp/pubsublite/PubsubLiteUnboundedSource.java | 126 --------
...zer.java => SerializableSubscriberFactory.java} | 17 +-
.../io/gcp/pubsublite/SerializableSupplier.java | 2 +-
.../sdk/io/gcp/pubsublite/SubscribeTransform.java | 124 ++++++++
.../sdk/io/gcp/pubsublite/SubscriberOptions.java | 42 ++-
...etFinalizer.java => SubscriptionPartition.java} | 19 +-
...dCoder.java => SubscriptionPartitionCoder.java} | 36 ++-
...er.java => SubscriptionPartitionProcessor.java} | 14 +-
... => SubscriptionPartitionProcessorFactory.java} | 14 +-
.../SubscriptionPartitionProcessorImpl.java | 152 ++++++++++
.../sdk/io/gcp/pubsublite/TopicBacklogReader.java | 33 +-
.../io/gcp/pubsublite/TopicBacklogReaderImpl.java | 80 ++---
.../gcp/pubsublite/TopicBacklogReaderSettings.java | 31 +-
.../gcp/pubsublite/TranslatingPullSubscriber.java | 55 ----
.../beam/sdk/io/gcp/pubsublite/UuidCoder.java | 7 +-
.../io/gcp/testing/BigtableEmulatorWrapper.java | 75 -----
.../beam/sdk/io/gcp/testing/BigtableTestUtils.java | 153 ----------
.../beam/sdk/io/gcp/testing/BigtableUtils.java | 49 +++
.../bigtable/BeamRowToBigtableMutationTest.java | 14 +-
.../gcp/bigtable/BigtableRowToBeamRowFlatTest.java | 14 +-
.../io/gcp/bigtable/BigtableRowToBeamRowTest.java | 17 +-
.../{TestUtils.java => BigtableTestUtils.java} | 50 +++-
.../sdk/io/gcp/bigtable/CellValueParserTest.java | 13 +-
.../gcp/pubsublite/OffsetByteRangeTrackerTest.java | 158 ++++++++++
.../gcp/pubsublite/OffsetCheckpointMarkTest.java | 81 -----
.../PerSubscriptionPartitionSdfTest.java | 189 ++++++++++++
.../pubsublite/PubsubLiteUnboundedReaderTest.java | 330 --------------------
.../SubscriptionPartitionProcessorImplTest.java | 227 ++++++++++++++
.../gcp/pubsublite/TopicBacklogReaderImplTest.java | 129 ++------
sdks/python/apache_beam/dataframe/io.py | 258 ++++++++++++----
sdks/python/apache_beam/dataframe/io_test.py | 88 +++++-
sdks/python/apache_beam/io/concat_source.py | 9 +-
sdks/python/apache_beam/io/concat_source_test.py | 8 +
sdks/python/apache_beam/metrics/execution.pxd | 1 +
sdks/python/apache_beam/metrics/execution.py | 9 +-
sdks/python/apache_beam/transforms/core.py | 41 +--
sdks/python/apache_beam/version.py | 2 +-
sdks/python/container/common.gradle | 2 +
settings.gradle | 1 +
81 files changed, 3479 insertions(+), 1924 deletions(-)
create mode 100644 .test-infra/jenkins/job_LoadTests_SideInput_Flink_Go.groovy
create mode 100644 .test-infra/jenkins/job_LoadTests_SideInput_Go.groovy
create mode 100644 sdks/go/test/load/sideinput/sideinput.go
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableClientWrapper.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableIT.java
delete mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableTestUtils.java
copy sdks/java/{io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/OffsetFinalizer.java => extensions/sql/udf/build.gradle} (50%)
create mode 100644 sdks/java/extensions/sql/udf/src/main/java/org/apache/beam/sdk/extensions/sql/udf/AggregateFn.java
create mode 100644 sdks/java/extensions/sql/udf/src/main/java/org/apache/beam/sdk/extensions/sql/udf/ScalarFn.java
copy sdks/java/{io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/SerializableSupplier.java => extensions/sql/udf/src/main/java/org/apache/beam/sdk/extensions/sql/udf/package-info.java} (68%)
create mode 100644 sdks/java/harness/src/main/java/org/apache/beam/fn/harness/TransformProcessingThreadTracker.java
copy sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{OffsetFinalizer.java => InitialOffsetReader.java} (75%)
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/InitialOffsetReaderImpl.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/LimitingTopicBacklogReader.java
copy sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{OffsetFinalizer.java => OffsetByteProgress.java} (68%)
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/OffsetByteRangeTracker.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/OffsetCheckpointMark.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PerSubscriptionPartitionSdf.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteUnboundedReader.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteUnboundedSource.java
copy sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{OffsetFinalizer.java => SerializableSubscriberFactory.java} (65%)
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/SubscribeTransform.java
copy sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{OffsetFinalizer.java => SubscriptionPartition.java} (64%)
copy sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{UuidCoder.java => SubscriptionPartitionCoder.java} (50%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{OffsetFinalizer.java => SubscriptionPartitionProcessor.java} (74%)
copy sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{SerializableSupplier.java => SubscriptionPartitionProcessorFactory.java} (60%)
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/SubscriptionPartitionProcessorImpl.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/TranslatingPullSubscriber.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/BigtableEmulatorWrapper.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/BigtableTestUtils.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/BigtableUtils.java
rename sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/{TestUtils.java => BigtableTestUtils.java} (71%)
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/OffsetByteRangeTrackerTest.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/OffsetCheckpointMarkTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/PerSubscriptionPartitionSdfTest.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteUnboundedReaderTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/SubscriptionPartitionProcessorImplTest.java